Error when calling trainer.fit(vad_model)
State: closed | Pull request: no
Labels: name: bug

Hello. 
I have successfully trained VAD according to your tutorial with my background data set. However, when I start training VAD with a background sample length of 0.1s, I get an error. Please tell me which parameter needs to be changed to avoid this.

`trainer.fit(vad_model)`

```

LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]
[NeMo I 2021-06-18 12:37:26 modelPT:751] Optimizer config = SGD (
    Parameter Group 0
        dampening: 0
        lr: 0.01
        momentum: 0.9
        nesterov: False
        weight_decay: 0.001
    )
[NeMo I 2021-06-18 12:37:26 lr_scheduler:621] Scheduler "<nemo.core.optim.lr_scheduler.PolynomialHoldDecayAnnealing object at 0x7fcb9347f2e0>" 
    will be used during training (effective maximum steps = 2370) - 
    Parameters : 
    (power: 2.0
    warmup_ratio: 0.05
    hold_ratio: 0.45
    min_lr: 0.001
    last_epoch: -1
    max_steps: 2370
    )

  | Name              | Type                         | Params
-------------------------------------------------------------------
0 | spec_augmentation | SpectrogramAugmentation      | 0     
1 | preprocessor      | AudioToMFCCPreprocessor      | 0     
2 | encoder           | ConvASREncoder               | 88.9 K
3 | decoder           | ConvASRDecoderClassification | 258   
4 | loss              | CrossEntropyLoss             | 0     
5 | _accuracy         | TopKClassificationAccuracy   | 0     
-------------------------------------------------------------------
89.2 K    Trainable params
0         Non-trainable params
89.2 K    Total params
0.357     Total estimated model params size (MB)

  | Name              | Type                         | Params
-------------------------------------------------------------------
0 | spec_augmentation | SpectrogramAugmentation      | 0     
1 | preprocessor      | AudioToMFCCPreprocessor      | 0     
2 | encoder           | ConvASREncoder               | 88.9 K
3 | decoder           | ConvASRDecoderClassification | 258   
4 | loss              | CrossEntropyLoss             | 0     
5 | _accuracy         | TopKClassificationAccuracy   | 0     
-------------------------------------------------------------------
89.2 K    Trainable params
0         Non-trainable params
89.2 K    Total params
0.357     Total estimated model params size (MB)
[NeMo W 2021-06-18 12:37:27 nemo_logging:349] /opt/conda/lib/python3.8/site-packages/pytorch_lightning/utilities/distributed.py:69: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 24 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
      warnings.warn(*args, **kwargs)
    
[NeMo W 2021-06-18 12:37:27 nemo_logging:349] /opt/conda/lib/python3.8/site-packages/pytorch_lightning/utilities/distributed.py:69: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 24 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
      warnings.warn(*args, **kwargs)


ValueError                                Traceback (most recent call last)
<ipython-input-185-6abb1ca1f787> in <module>
----> 1 trainer.fit(vad_model)

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloader, val_dataloaders, datamodule)
    456         )
    457 
--> 458         self._run(model)
    459 
    460         assert self.state.stopped

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in _run(self, model)
    754 
    755         # dispatch `start_training` or `start_evaluating` or `start_predicting`
--> 756         self.dispatch()
    757 
    758         # plugin will finalized fitting (e.g. ddp_spawn will load trained model)

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in dispatch(self)
    795             self.accelerator.start_predicting(self)
    796         else:
--> 797             self.accelerator.start_training(self)
    798 
    799     def run_stage(self):

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py in start_training(self, trainer)
     94 
     95     def start_training(self, trainer: 'pl.Trainer') -> None:
---> 96         self.training_type_plugin.start_training(trainer)
     97 
     98     def start_evaluating(self, trainer: 'pl.Trainer') -> None:

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in start_training(self, trainer)
    142     def start_training(self, trainer: 'pl.Trainer') -> None:
    143         # double dispatch to initiate the training loop
--> 144         self._results = trainer.run_stage()
    145 
    146     def start_evaluating(self, trainer: 'pl.Trainer') -> None:

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in run_stage(self)
    805         if self.predicting:
    806             return self.run_predict()
--> 807         return self.run_train()
    808 
    809     def _pre_training_routine(self):

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in run_train(self)
    867                 with self.profiler.profile("run_training_epoch"):
    868                     # run train epoch
--> 869                     self.train_loop.run_training_epoch()
    870 
    871                 if self.max_steps and self.max_steps <= self.global_step:

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in run_training_epoch(self)
    497             # ------------------------------------
    498             with self.trainer.profiler.profile("run_training_batch"):
--> 499                 batch_output = self.run_training_batch(batch, batch_idx, dataloader_idx)
    500 
    501             # when returning -1 from train_step, we end epoch early

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in run_training_batch(self, batch, batch_idx, dataloader_idx)
    736 
    737                         # optimizer step
--> 738                         self.optimizer_step(optimizer, opt_idx, batch_idx, train_step_and_backward_closure)
    739                         if len(self.trainer.optimizers) > 1:
    740                             # revert back to previous state

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in optimizer_step(self, optimizer, opt_idx, batch_idx, train_step_and_backward_closure)
    432 
    433         # model hook
--> 434         model_ref.optimizer_step(
    435             self.trainer.current_epoch,
    436             batch_idx,

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/core/lightning.py in optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx, optimizer_closure, on_tpu, using_native_amp, using_lbfgs)
   1401 
   1402         """
-> 1403         optimizer.step(closure=optimizer_closure)
   1404 
   1405     def optimizer_zero_grad(self, epoch: int, batch_idx: int, optimizer: Optimizer, optimizer_idx: int):

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py in step(self, closure, *args, **kwargs)
    212             profiler_name = f"optimizer_step_and_closure_{self._optimizer_idx}"
    213 
--> 214         self.__optimizer_step(*args, closure=closure, profiler_name=profiler_name, **kwargs)
    215         self._total_optimizer_step_calls += 1
    216 

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py in __optimizer_step(self, closure, profiler_name, **kwargs)
    132 
    133         with trainer.profiler.profile(profiler_name):
--> 134             trainer.accelerator.optimizer_step(optimizer, self._optimizer_idx, lambda_closure=closure, **kwargs)
    135 
    136     def step(self, *args, closure: Optional[Callable] = None, **kwargs):

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py in optimizer_step(self, optimizer, opt_idx, lambda_closure, **kwargs)
    327         )
    328         if make_optimizer_step:
--> 329             self.run_optimizer_step(optimizer, opt_idx, lambda_closure, **kwargs)
    330         self.precision_plugin.post_optimizer_step(optimizer, opt_idx)
    331         self.training_type_plugin.post_optimizer_step(optimizer, opt_idx, **kwargs)

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py in run_optimizer_step(self, optimizer, optimizer_idx, lambda_closure, **kwargs)
    334         self, optimizer: Optimizer, optimizer_idx: int, lambda_closure: Callable, **kwargs: Any
    335     ) -> None:
--> 336         self.training_type_plugin.optimizer_step(optimizer, lambda_closure=lambda_closure, **kwargs)
    337 
    338     def optimizer_zero_grad(self, current_epoch: int, batch_idx: int, optimizer: Optimizer, opt_idx: int) -> None:

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in optimizer_step(self, optimizer, lambda_closure, **kwargs)
    191 
    192     def optimizer_step(self, optimizer: torch.optim.Optimizer, lambda_closure: Callable, **kwargs):
--> 193         optimizer.step(closure=lambda_closure, **kwargs)
    194 
    195     @property

/opt/conda/lib/python3.8/site-packages/torch/optim/lr_scheduler.py in wrapper(*args, **kwargs)
     63                 instance._step_count += 1
     64                 wrapped = func.__get__(instance, cls)
---> 65                 return wrapped(*args, **kwargs)
     66 
     67             # Note that the returned function here is no longer a bound method,

/opt/conda/lib/python3.8/site-packages/torch/optim/optimizer.py in wrapper(*args, **kwargs)
     86                 profile_name = "Optimizer.step#{}.step".format(obj.__class__.__name__)
     87                 with torch.autograd.profiler.record_function(profile_name):
---> 88                     return func(*args, **kwargs)
     89             return wrapper
     90 

/opt/conda/lib/python3.8/site-packages/torch/autograd/grad_mode.py in decorate_context(*args, **kwargs)
     26         def decorate_context(*args, **kwargs):
     27             with self.__class__():
---> 28                 return func(*args, **kwargs)
     29         return cast(F, decorate_context)
     30 

/opt/conda/lib/python3.8/site-packages/torch/optim/sgd.py in step(self, closure)
     85         if closure is not None:
     86             with torch.enable_grad():
---> 87                 loss = closure()
     88 
     89         for group in self.param_groups:

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in train_step_and_backward_closure()
    730 
    731                         def train_step_and_backward_closure():
--> 732                             result = self.training_step_and_backward(
    733                                 split_batch, batch_idx, opt_idx, optimizer, self.trainer.hiddens
    734                             )

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in training_step_and_backward(self, split_batch, batch_idx, opt_idx, optimizer, hiddens)
    821         with self.trainer.profiler.profile("training_step_and_backward"):
    822             # lightning module hook
--> 823             result = self.training_step(split_batch, batch_idx, opt_idx, hiddens)
    824             self._curr_step_result = result
    825 

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in training_step(self, split_batch, batch_idx, opt_idx, hiddens)
    288             model_ref._results = Result()
    289             with self.trainer.profiler.profile("training_step"):
--> 290                 training_step_output = self.trainer.accelerator.training_step(args)
    291                 self.trainer.accelerator.post_training_step()
    292 

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py in training_step(self, args)
    202 
    203         with self.precision_plugin.train_step_context(), self.training_type_plugin.train_step_context():
--> 204             return self.training_type_plugin.training_step(*args)
    205 
    206     def post_training_step(self) -> None:

/opt/conda/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in training_step(self, *args, **kwargs)
    153 
    154     def training_step(self, *args, **kwargs):
--> 155         return self.lightning_module.training_step(*args, **kwargs)
    156 
    157     def post_training_step(self):

/workspace/nistrian/NeMo/nemo/utils/model_utils.py in wrap_training_step(wrapped, instance, args, kwargs)
    337 @wrapt.decorator
    338 def wrap_training_step(wrapped, instance: pl.LightningModule, args, kwargs):
--> 339     output_dict = wrapped(*args, **kwargs)
    340 
    341     if isinstance(output_dict, dict) and output_dict is not None and 'log' in output_dict:

/workspace/nistrian/NeMo/nemo/collections/asr/models/classification_models.py in training_step(self, batch, batch_nb)
    470         self.training_step_end()
    471         audio_signal, audio_signal_len, labels, labels_len = batch
--> 472         logits = self.forward(input_signal=audio_signal, input_signal_length=audio_signal_len)
    473         loss_value = self.loss(logits=logits, labels=labels)
    474 

/workspace/nistrian/NeMo/nemo/core/classes/common.py in __call__(self, wrapped, instance, args, kwargs)
    790 
    791         # Call the method - this can be forward, or any other callable method
--> 792         outputs = wrapped(*args, **kwargs)
    793 
    794         instance._attach_and_validate_output_types(

/workspace/nistrian/NeMo/nemo/collections/asr/models/classification_models.py in forward(self, input_signal, input_signal_length)
    547     @typecheck()
    548     def forward(self, input_signal, input_signal_length):
--> 549         logits = super().forward(input_signal=input_signal, input_signal_length=input_signal_length)
    550         return logits
    551 

/workspace/nistrian/NeMo/nemo/collections/asr/models/classification_models.py in forward(self, input_signal, input_signal_length)
    145         # Spec augment is not applied during evaluation/testing
    146         if self.spec_augmentation is not None and self.training:
--> 147             processed_signal = self.spec_augmentation(input_spec=processed_signal, length=processed_signal_len)
    148         encoded, encoded_len = self.encoder(audio_signal=processed_signal, length=processed_signal_len)
    149         logits = self.decoder(encoder_output=encoded)

/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1049         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1050                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1051             return forward_call(*input, **kwargs)
   1052         # Do not call functions when jit is used
   1053         full_backward_hooks, non_full_backward_hooks = [], []

/workspace/nistrian/NeMo/nemo/core/classes/common.py in __call__(self, wrapped, instance, args, kwargs)
    790 
    791         # Call the method - this can be forward, or any other callable method
--> 792         outputs = wrapped(*args, **kwargs)
    793 
    794         instance._attach_and_validate_output_types(

/workspace/nistrian/NeMo/nemo/collections/asr/modules/audio_preprocessing.py in forward(self, input_spec, length)
    491     @typecheck()
    492     def forward(self, input_spec, length=None):
--> 493         augmented_spec = self.spec_cutout(input_spec=input_spec)
    494 
    495         # To run the Numba kernel, correct numba version is required as well as

/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1049         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1050                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1051             return forward_call(*input, **kwargs)
   1052         # Do not call functions when jit is used
   1053         full_backward_hooks, non_full_backward_hooks = [], []

/workspace/nistrian/NeMo/nemo/core/classes/common.py in __call__(self, wrapped, instance, args, kwargs)
    790 
    791         # Call the method - this can be forward, or any other callable method
--> 792         outputs = wrapped(*args, **kwargs)
    793 
    794         instance._attach_and_validate_output_types(

/opt/conda/lib/python3.8/site-packages/torch/autograd/grad_mode.py in decorate_context(*args, **kwargs)
     26         def decorate_context(*args, **kwargs):
     27             with self.__class__():
---> 28                 return func(*args, **kwargs)
     29         return cast(F, decorate_context)
     30 

/workspace/nistrian/NeMo/nemo/collections/asr/parts/submodules/spectr_augment.py in forward(self, input_spec)
    143             for i in range(self.rect_masks):
    144                 rect_x = self._rng.randint(0, sh[1] - self.rect_freq)
--> 145                 rect_y = self._rng.randint(0, sh[2] - self.rect_time)
    146 
    147                 w_x = self._rng.randint(0, self.rect_freq)

/opt/conda/lib/python3.8/random.py in randint(self, a, b)
    246         """
    247 
--> 248         return self.randrange(a, b+1)
    249 
    250     def _randbelow_with_getrandbits(self, n):

/opt/conda/lib/python3.8/random.py in randrange(self, start, stop, step, _int)
    224             return istart + self._randbelow(width)
    225         if step == 1:
--> 226             raise ValueError("empty range for randrange() (%d, %d, %d)" % (istart, istop, width))
    227 
    228         # Non-unit step argument supplied.

ValueError: empty range for randrange() (0, -13, -13)
```