Error during inference of SpeechT5 model for TTS
Author: VafaKnm
Pull request: no
Labels: name: Audio

Hi!
I trained a TTS model for Persian language using SpeechT5 model. I followed the steps of this link:
https://huggingface.co/learn/audio-course/en/chapter6/fine-tuning

But during inference, i got an error which i can't understand its reason. This is my code for inference:
```
tokenizer = SpeechT5Tokenizer.from_pretrained("./tokenizers/spm_tokenizer/m.model", bos_token="[s]", eos_token="[e]",  unk_token="[UNK]", pad_token="[PAD]")
feature_extractor = SpeechT5FeatureExtractor()
processor = SpeechT5Processor(feature_extractor=feature_extractor, tokenizer=tokenizer)

config = SpeechT5Config.from_json_file('/home/vafa/speechT5_tts_output_partial/checkpoint-19000/config.json')
model = SpeechT5ForTextToSpeech(config)

state_dict = torch.load('/home/vafa/speechT5_tts_output_partial/checkpoint-19000/pytorch_model.bin')
model.load_state_dict(state_dict)

text = "سلام حالت خوبه"
inputs = processor(text=text, return_tensors="pt")

speaker_embeddings = torch.tensor(train_ds[0]['speaker_embeddings']).unsqueeze(0)

vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
speech = model.generate(inputs["input_ids"], speaker_embeddings=speaker_embeddings, attention_mask=inputs["attention_mask"], vocoder=vocoder)
```

And this is the error i got:
```
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In[14], line 1
----> 1 speech = model.generate(inputs["input_ids"], speaker_embeddings=speaker_embeddings, attention_mask=inputs["attention_mask"], vocoder=vocoder)
      2 speech

File ~/.local/lib/python3.10/site-packages/torch/utils/_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File ~/.local/lib/python3.10/site-packages/transformers/models/speecht5/modeling_speecht5.py:2834, in SpeechT5ForTextToSpeech.generate(self, input_ids, attention_mask, speaker_embeddings, threshold, minlenratio, maxlenratio, vocoder, output_cross_attentions, return_output_lengths, **kwargs)
   2829         else:
   2830             raise ValueError(
   2831                 "The first dimension of speaker_embeddings must be either 1 or the same as batch_size."
   2832             )
-> 2834 return _generate_speech(
   2835     self,
   2836     input_ids,
   2837     speaker_embeddings,
   2838     attention_mask,
   2839     threshold,
   2840     minlenratio,
   2841     maxlenratio,
   2842     vocoder,
   2843     output_cross_attentions,
   2844     return_output_lengths,
   2845 )

File ~/.local/lib/python3.10/site-packages/transformers/models/speecht5/modeling_speecht5.py:2518, in _generate_speech(model, input_values, speaker_embeddings, attention_mask, threshold, minlenratio, maxlenratio, vocoder, output_cross_attentions, return_output_lengths)
   2516 decoder_hidden_states = model.speecht5.decoder.prenet(output_sequence, speaker_embeddings)
   2517 # Run the decoder layers on the last element of the prenet output.
-> 2518 decoder_out = model.speecht5.decoder.wrapped_decoder(
   2519     hidden_states=decoder_hidden_states[:, -1:],
   2520     attention_mask=None,
   2521     encoder_hidden_states=encoder_last_hidden_state,
   2522     encoder_attention_mask=encoder_attention_mask,
   2523     past_key_values=past_key_values,
   2524     use_cache=True,
   2525     output_attentions=output_cross_attentions,
   2526     return_dict=True,
   2527 )
   2529 if output_cross_attentions:
   2530     cross_attentions.append(torch.cat(decoder_out.cross_attentions, dim=0))

File ~/.local/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.10/site-packages/transformers/models/speecht5/modeling_speecht5.py:1640, in SpeechT5Decoder.forward(self, hidden_states, attention_mask, encoder_hidden_states, encoder_attention_mask, head_mask, cross_attn_head_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
   1637 if skip_the_layer and not deepspeed_zero3_is_enabled:
   1638     continue
-> 1640 past_key_value = past_key_values[idx] if past_key_values is not None else None
   1642 if self.gradient_checkpointing and self.training:
   1643     layer_outputs = self._gradient_checkpointing_func(
   1644         decoder_layer.__call__,
   1645         hidden_states,
   (...)
   1653         use_cache,
   1654     )

IndexError: tuple index out of range
```

Can anyone help?

Comments:
cc @ylacombe @sanchit-gandhi 

Sorry... It's been almost a week since I created this issue but I still haven't received a response. I will be very grateful if anyone can help.

cc @kamilakesbi too