Cannot finetune Univnet on the spectrogram of MixerTTS
State: closed | Pull request: no
Labels: name: bug

I trained a MixerTTS-X model, and would like to finetune a Univnet model with the spectrograms generated by MixerTTS.
I followed the tutorial about finetuning FastPitch and Hifigan and changed the code related to generating spectrograms to:
```python
    fn=os.path.split(r['audio_filepath'])[1]
    audio = load_wav(r["audio_filepath"])
    audio = torch.from_numpy(audio).unsqueeze(0).to(device)
    audio_len = torch.tensor(audio.shape[1], dtype=torch.long, device=device).unsqueeze(0)
    
    with torch.no_grad():
        if "normalized_text" in r:
            text = spec_model.parse(r["normalized_text"], normalize=False)
            raw_texts=[r["normalized_text"]]
        else:
            text = spec_model.parse(r['text'])
            raw_texts=[r["text"]]
        
        text_len = torch.tensor(text.shape[-1], dtype=torch.long, device=device).unsqueeze(0)
    
        spect, spect_len = spec_model.preprocessor(input_signal=audio, length=audio_len)

        attn_prior = torch.from_numpy(
          beta_binomial_interpolator(spect_len.item(), text_len.item())
        ).unsqueeze(0).to(text.device)
        
        if hasattr(spec_model.cfg,"lm_model"):
            lm_model_tokenizer = spec_model._get_lm_model_tokenizer(spec_model.cfg.lm_model)
            lm_padding_value = lm_model_tokenizer._convert_token_to_id('<pad>')
            lm_space_value = lm_model_tokenizer._convert_token_to_id('▁')

            if spec_model.text_normalizer_call is not None:
                raw_texts = [spec_model.text_normalizer_call(t, **spec_model.text_normalizer_call_kwargs) for t in raw_texts]

            preprocess_texts_as_tts_input = [spec_model.tokenizer.text_preprocessing_func(t) for t in raw_texts]
            lm_tokens_as_ids_list = [
                lm_model_tokenizer.encode(t, add_special_tokens=False) for t in preprocess_texts_as_tts_input
            ]

            if spec_model.tokenizer.pad_with_space:
                lm_tokens_as_ids_list = [[lm_space_value] + t + [lm_space_value] for t in lm_tokens_as_ids_list]

            lm_tokens = torch.full(
                (len(lm_tokens_as_ids_list), max([len(t) for t in lm_tokens_as_ids_list])),
                fill_value=lm_padding_value,
                device=device,
            )
            for i, lm_tokens_i in enumerate(lm_tokens_as_ids_list):
                lm_tokens[i, : len(lm_tokens_i)] = torch.tensor(lm_tokens_i, device=device)
        
        spectrogram = spec_model.forward(
          text=text, 
          text_len=text_len, 
          spect=spect, 
          spect_len=spect_len, 
          attn_prior=attn_prior,
          lm_tokens=lm_tokens
        )[0]
        
        save_path = os.path.join(save_dir , f"mel_{fn}.npy")
        np.save(save_path, spectrogram[0].to('cpu').numpy())
```
Then I started the finetuning with the command:
```examples/tts/univnet_finetune.py --config-name=univnet.yaml model.optim.lr=0.00001 train_dataset=/root/autodl-tmp/ryanspeech.train_mel.json validation_datasets=/root/autodl-tmp/ryanspeech.val_mel.json exp_manager.exp_dir=/root/autodl-tmp/nemo-training/ryan_22k exp_manager.resume_if_exists=True exp_manager.resume_ignore_no_checkpoint=True +init_from_nemo_model=/root/autodl-tmp/tts_en_libritts_multispeaker_univnet.nemo model/train_ds=train_ds_finetune model/validation_ds=val_ds_finetune model.train_ds.dataloader_params.batch_size=32 trainer.check_val_every_n_epoch=10```
And the following error occurred:
```
Traceback (most recent call last):
  File "examples/tts/univnet-finetune.py", line 32, in main
    trainer.fit(model)
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 696, in fit
    self._call_and_handle_interrupt(
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 648, in _call_and_handle_interrupt
    return self.strategy.launcher.launch(trainer_fn, *args, trainer=self, **kwargs)
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 93, in launch
    return function(*args, **kwargs)
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 737, in _fit_impl
    results = self._run(model, ckpt_path=self.ckpt_path)
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1168, in _run
    results = self._run_stage()
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1254, in _run_stage
    return self._run_train()
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1276, in _run_train
    self._run_sanity_check()
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1345, in _run_sanity_check
    val_loop.run()
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
    self.advance(*args, **kwargs)
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py", line 155, in advance
    dl_outputs = self.epoch_loop.run(self._data_fetcher, dl_max_batches, kwargs)
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
    self.advance(*args, **kwargs)
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 127, in advance
    batch = next(data_fetcher)
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 185, in __next__
    return self.fetching_function()
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 264, in fetching_function
    self._fetch_next_batch(self.dataloader_iter)
  File "/root/miniconda3/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 278, in _fetch_next_batch
    batch = next(iterator)
  File "/root/miniconda3/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 521, in __next__
    data = self._next_data()
  File "/root/miniconda3/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1203, in _next_data
    return self._process_data(data)
  File "/root/miniconda3/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1229, in _process_data
    data.reraise()
  File "/root/miniconda3/lib/python3.8/site-packages/torch/_utils.py", line 434, in reraise
    raise exception
ValueError: Caught ValueError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/root/miniconda3/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py", line 287, in _worker_loop
    data = fetcher.fetch(index)
  File "/root/miniconda3/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 49, in fetch
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/root/miniconda3/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 49, in <listcomp>
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/root/NeMo/nemo/collections/tts/torch/data.py", line 973, in __getitem__
    start = random.randint(0, mel.shape[1] - frames - 1)
  File "/root/miniconda3/lib/python3.8/random.py", line 248, in randint
    return self.randrange(a, b+1)
  File "/root/miniconda3/lib/python3.8/random.py", line 226, in randrange
    raise ValueError("empty range for randrange() (%d, %d, %d)" % (istart, istop, width))
ValueError: empty range for randrange() (0, -178, -178)
```
Similar error has been discussed in #3628, but still not sure how to solve it.
Any help please?