from typing import Dict import numpy as np import torch from torch import nn def numpy_to_torch(np_array, dtype, cuda=False, device="cpu"): if cuda: device = "cuda" if np_array is None: return None tensor = torch.as_tensor(np_array, dtype=dtype, device=device) return tensor def compute_style_mel(style_wav, ap, cuda=False, device="cpu"): if cuda: device = "cuda" style_mel = torch.FloatTensor( ap.melspectrogram(ap.load_wav(style_wav, sr=ap.sample_rate)), device=device, ).unsqueeze(0) return style_mel def run_model_torch( model: nn.Module, inputs: torch.Tensor, speaker_id: int = None, style_mel: torch.Tensor = None, style_text: str = None, d_vector: torch.Tensor = None, language_id: torch.Tensor = None, ) -> Dict: """Run a torch model for inference. It does not support batch inference. Args: model (nn.Module): The model to run inference. inputs (torch.Tensor): Input tensor with character ids. speaker_id (int, optional): Input speaker ids for multi-speaker models. Defaults to None. style_mel (torch.Tensor, optional): Spectrograms used for voice styling . Defaults to None. d_vector (torch.Tensor, optional): d-vector for multi-speaker models . Defaults to None. Returns: Dict: model outputs. """ input_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device) if hasattr(model, "module"): _func = model.module.inference else: _func = model.inference outputs = _func( inputs, aux_input={ "x_lengths": input_lengths, "speaker_ids": speaker_id, "d_vectors": d_vector, "style_mel": style_mel, "style_text": style_text, "language_ids": language_id, }, ) return outputs def trim_silence(wav, ap): return wav[: ap.find_endpoint(wav)] def inv_spectrogram(postnet_output, ap, CONFIG): if CONFIG.model.lower() in ["tacotron"]: wav = ap.inv_spectrogram(postnet_output.T) else: wav = ap.inv_melspectrogram(postnet_output.T) return wav def id_to_torch(aux_id, cuda=False, device="cpu"): if cuda: device = "cuda" if aux_id is not None: aux_id = np.asarray(aux_id) aux_id = torch.from_numpy(aux_id).to(device) return aux_id def embedding_to_torch(d_vector, cuda=False, device="cpu"): if cuda: device = "cuda" if d_vector is not None: d_vector = np.asarray(d_vector) d_vector = torch.from_numpy(d_vector).type(torch.FloatTensor) d_vector = d_vector.squeeze().unsqueeze(0).to(device) return d_vector # TODO: perform GL with pytorch for batching def apply_griffin_lim(inputs, input_lens, CONFIG, ap): """Apply griffin-lim to each sample iterating throught the first dimension. Args: inputs (Tensor or np.Array): Features to be converted by GL. First dimension is the batch size. input_lens (Tensor or np.Array): 1D array of sample lengths. CONFIG (Dict): TTS config. ap (AudioProcessor): TTS audio processor. """ wavs = [] for idx, spec in enumerate(inputs): wav_len = (input_lens[idx] * ap.hop_length) - ap.hop_length # inverse librosa padding wav = inv_spectrogram(spec, ap, CONFIG) # assert len(wav) == wav_len, f" [!] wav lenght: {len(wav)} vs expected: {wav_len}" wavs.append(wav[:wav_len]) return wavs def synthesis( model, text, CONFIG, use_cuda, speaker_id=None, style_wav=None, style_text=None, use_griffin_lim=False, do_trim_silence=False, d_vector=None, language_id=None, ): """Synthesize voice for the given text using Griffin-Lim vocoder or just compute output features to be passed to the vocoder model. Args: model (TTS.tts.models): The TTS model to synthesize audio with. text (str): The input text to convert to speech. CONFIG (Coqpit): Model configuration. use_cuda (bool): Enable/disable CUDA. speaker_id (int): Speaker ID passed to the speaker embedding layer in multi-speaker model. Defaults to None. style_wav (str | Dict[str, float]): Path or tensor to/of a waveform used for computing the style embedding based on GST or Capacitron. Defaults to None, meaning that Capacitron models will sample from the prior distribution to generate random but realistic prosody. style_text (str): Transcription of style_wav for Capacitron models. Defaults to None. enable_eos_bos_chars (bool): enable special chars for end of sentence and start of sentence. Defaults to False. do_trim_silence (bool): trim silence after synthesis. Defaults to False. d_vector (torch.Tensor): d-vector for multi-speaker models in share :math:`[1, D]`. Defaults to None. language_id (int): Language ID passed to the language embedding layer in multi-langual model. Defaults to None. """ # device device = next(model.parameters()).device if use_cuda: device = "cuda" # GST or Capacitron processing # TODO: need to handle the case of setting both gst and capacitron to true somewhere style_mel = None if CONFIG.has("gst") and CONFIG.gst and style_wav is not None: if isinstance(style_wav, dict): style_mel = style_wav else: style_mel = compute_style_mel(style_wav, model.ap, device=device) if CONFIG.has("capacitron_vae") and CONFIG.use_capacitron_vae and style_wav is not None: style_mel = compute_style_mel(style_wav, model.ap, device=device) style_mel = style_mel.transpose(1, 2) # [1, time, depth] language_name = None if language_id is not None: language = [k for k, v in model.language_manager.name_to_id.items() if v == language_id] assert len(language) == 1, "language_id must be a valid language" language_name = language[0] # convert text to sequence of token IDs text_inputs = np.asarray( model.tokenizer.text_to_ids(text, language=language_name), dtype=np.int32, ) # pass tensors to backend if speaker_id is not None: speaker_id = id_to_torch(speaker_id, device=device) if d_vector is not None: d_vector = embedding_to_torch(d_vector, device=device) if language_id is not None: language_id = id_to_torch(language_id, device=device) if not isinstance(style_mel, dict): # GST or Capacitron style mel style_mel = numpy_to_torch(style_mel, torch.float, device=device) if style_text is not None: style_text = np.asarray( model.tokenizer.text_to_ids(style_text, language=language_id), dtype=np.int32, ) style_text = numpy_to_torch(style_text, torch.long, device=device) style_text = style_text.unsqueeze(0) text_inputs = numpy_to_torch(text_inputs, torch.long, device=device) text_inputs = text_inputs.unsqueeze(0) # synthesize voice outputs = run_model_torch( model, text_inputs, speaker_id, style_mel, style_text, d_vector=d_vector, language_id=language_id, ) model_outputs = outputs["model_outputs"] model_outputs = model_outputs[0].data.cpu().numpy() alignments = outputs["alignments"] # convert outputs to numpy # plot results wav = None model_outputs = model_outputs.squeeze() if model_outputs.ndim == 2: # [T, C_spec] if use_griffin_lim: wav = inv_spectrogram(model_outputs, model.ap, CONFIG) # trim silence if do_trim_silence: wav = trim_silence(wav, model.ap) else: # [T,] wav = model_outputs return_dict = { "wav": wav, "alignments": alignments, "text_inputs": text_inputs, "outputs": outputs, } return return_dict def transfer_voice( model, CONFIG, use_cuda, reference_wav, speaker_id=None, d_vector=None, reference_speaker_id=None, reference_d_vector=None, do_trim_silence=False, use_griffin_lim=False, ): """Synthesize voice for the given text using Griffin-Lim vocoder or just compute output features to be passed to the vocoder model. Args: model (TTS.tts.models): The TTS model to synthesize audio with. CONFIG (Coqpit): Model configuration. use_cuda (bool): Enable/disable CUDA. reference_wav (str): Path of reference_wav to be used to voice conversion. speaker_id (int): Speaker ID passed to the speaker embedding layer in multi-speaker model. Defaults to None. d_vector (torch.Tensor): d-vector for multi-speaker models in share :math:`[1, D]`. Defaults to None. reference_speaker_id (int): Reference Speaker ID passed to the speaker embedding layer in multi-speaker model. Defaults to None. reference_d_vector (torch.Tensor): Reference d-vector for multi-speaker models in share :math:`[1, D]`. Defaults to None. enable_eos_bos_chars (bool): enable special chars for end of sentence and start of sentence. Defaults to False. do_trim_silence (bool): trim silence after synthesis. Defaults to False. """ # device device = next(model.parameters()).device if use_cuda: device = "cuda" # pass tensors to backend if speaker_id is not None: speaker_id = id_to_torch(speaker_id, device=device) if d_vector is not None: d_vector = embedding_to_torch(d_vector, device=device) if reference_d_vector is not None: reference_d_vector = embedding_to_torch(reference_d_vector, device=device) # load reference_wav audio reference_wav = embedding_to_torch( model.ap.load_wav( reference_wav, sr=model.args.encoder_sample_rate if model.args.encoder_sample_rate else model.ap.sample_rate ), device=device, ) if hasattr(model, "module"): _func = model.module.inference_voice_conversion else: _func = model.inference_voice_conversion model_outputs = _func(reference_wav, speaker_id, d_vector, reference_speaker_id, reference_d_vector) # convert outputs to numpy # plot results wav = None model_outputs = model_outputs.squeeze() if model_outputs.ndim == 2: # [T, C_spec] if use_griffin_lim: wav = inv_spectrogram(model_outputs, model.ap, CONFIG) # trim silence if do_trim_silence: wav = trim_silence(wav, model.ap) else: # [T,] wav = model_outputs return wav