Repository: espnet/espnet

Issue:
real-time TTS demo - using reference speech
I have been trying to use the code below to generate an audio using reference speech.
I added 'use_teacher_forcing=True' in the Text2speech config to be able to use reference speech. But it doesn't seem to work.
The output on the console is provided below. Please help. Is there any way to use this code for voice conversion?

```
import soundfile
import logging
import espnet
import torch

lang = 'English'
tag = 'kan-bayashi/vctk_full_band_multi_spk_vits' #@param ["kan-bayashi/vctk_gst_tacotron2", "kan-bayashi/vctk_gst_transformer", "kan-bayashi/vctk_xvector_tacotron2", "kan-bayashi/vctk_xvector_transformer", "kan-bayashi/vctk_xvector_conformer_fastspeech2", "kan-bayashi/vctk_gst+xvector_tacotron2", "kan-bayashi/vctk_gst+xvector_transformer", "kan-bayashi/vctk_gst+xvector_conformer_fastspeech2", "kan-bayashi/vctk_multi_spk_vits", "kan-bayashi/vctk_full_band_multi_spk_vits", "kan-bayashi/libritts_xvector_transformer", "kan-bayashi/libritts_xvector_conformer_fastspeech2", "kan-bayashi/libritts_gst+xvector_transformer", "kan-bayashi/libritts_gst+xvector_conformer_fastspeech2", "kan-bayashi/libritts_xvector_vits"] {type:"string"}
vocoder_tag = "none" #@param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}

from espnet2.bin.tts_inference import Text2Speech
from espnet2.utils.types import str_or_none

text2speech = Text2Speech.from_pretrained(
    model_tag=str_or_none(tag),
    vocoder_tag=str_or_none(vocoder_tag),
    device="cuda",
    # Only for Tacotron 2 & Transformer
    threshold=0.5,
    # Only for Tacotron 2
    minlenratio=0.0,
    maxlenratio=10.0,
    use_att_constraint=False,
    backward_window=1,
    forward_window=3,
    # Only for FastSpeech & FastSpeech2 & VITS
    speed_control_alpha=1.0,
    # Only for VITS
    noise_scale=0.333,
    noise_scale_dur=0.333,
    use_teacher_forcing=True,
)


import glob
import os
import numpy as np
import kaldiio

# Get model directory path
from espnet_model_zoo.downloader import ModelDownloader
d = ModelDownloader()
model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])

# X-vector selection
spembs = None
if text2speech.use_spembs:
    xvector_ark = [p for p in glob.glob(f"{model_dir}/../../dump/**/spk_xvector.ark", recursive=True) if "tr" in p][0]
    xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
    spks = list(xvectors.keys())

    # randomly select speaker
    random_spk_idx = np.random.randint(0, len(spks))
    spk = spks[random_spk_idx]
    spembs = xvectors[spk]
    print(f"selected spk: {spk}")

# Speaker ID selection
sids = None
if text2speech.use_sids:
    spk2sid = glob.glob(f"{model_dir}/../../dump/**/spk2sid", recursive=True)[0]
    with open(spk2sid) as f:
        lines = [line.strip() for line in f.readlines()]
    sid2spk = {int(line.split()[1]): line.split()[0] for line in lines}
    
    # randomly select speaker
    sids = np.array(np.random.randint(1, len(sid2spk)))
    spk = sid2spk[int(sids)]
    print(f"selected spkid: {spk}")

# Reference speech selection for GST
speech = None
if text2speech.use_speech:
    # you can change here to load your own reference speech
    # e.g.
     import soundfile as sf
     speech, fs = sf.read("/home/chitralekha/espnet/egs/timit/asr2/M09_B1_C1_M2.wav")
     speech = torch.from_numpy(speech).float()
     print(f"selected wav")
    #speech = torch.randn(50000,) * 0.01
    

wav = text2speech("foobar",speech=speech, spembs=spembs, sids=sids)["wav"].cpu()
soundfile.write("out.wav", wav.numpy(), text2speech.fs, "PCM_16")

Output:
selected spkid: p341
selected wav
Traceback (most recent call last):
  File "vc_multispeaker.py", line 82, in <module>
    wav = text2speech("foobar",speech=speech, spembs=spembs, sids=sids)["wav"].cpu()
  File "/home/chitralekha/espent_latest/espnet/tools/venv/lib/python3.7/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
    return func(*args, **kwargs)
  File "/home/chitralekha/espent_latest/espnet/espnet2/bin/tts_inference.py", line 196, in __call__
    output_dict = self.model.inference(**batch, **cfg)
  File "/home/chitralekha/espent_latest/espnet/espnet2/tts/espnet_model.py", line 299, in inference
    output_dict = self.tts.inference(**input_dict, **decode_config)
TypeError: inference() got an unexpected keyword argument 'speech'
```

real-time TTS demo - using reference speech
I have been trying to use the code below to generate an audio using reference speech.
I added 'use_teacher_forcing=True' in the Text2speech config to be able to use reference speech. But it doesn't seem to work.
The output on the console is provided below. Please help. Is there any way to use this code for voice conversion?

```
import soundfile
import logging
import espnet
import torch

lang = 'English'
tag = 'kan-bayashi/vctk_full_band_multi_spk_vits' #@param ["kan-bayashi/vctk_gst_tacotron2", "kan-bayashi/vctk_gst_transformer", "kan-bayashi/vctk_xvector_tacotron2", "kan-bayashi/vctk_xvector_transformer", "kan-bayashi/vctk_xvector_conformer_fastspeech2", "kan-bayashi/vctk_gst+xvector_tacotron2", "kan-bayashi/vctk_gst+xvector_transformer", "kan-bayashi/vctk_gst+xvector_conformer_fastspeech2", "kan-bayashi/vctk_multi_spk_vits", "kan-bayashi/vctk_full_band_multi_spk_vits", "kan-bayashi/libritts_xvector_transformer", "kan-bayashi/libritts_xvector_conformer_fastspeech2", "kan-bayashi/libritts_gst+xvector_transformer", "kan-bayashi/libritts_gst+xvector_conformer_fastspeech2", "kan-bayashi/libritts_xvector_vits"] {type:"string"}
vocoder_tag = "none" #@param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}

from espnet2.bin.tts_inference import Text2Speech
from espnet2.utils.types import str_or_none

text2speech = Text2Speech.from_pretrained(
    model_tag=str_or_none(tag),
    vocoder_tag=str_or_none(vocoder_tag),
    device="cuda",
    # Only for Tacotron 2 & Transformer
    threshold=0.5,
    # Only for Tacotron 2
    minlenratio=0.0,
    maxlenratio=10.0,
    use_att_constraint=False,
    backward_window=1,
    forward_window=3,
    # Only for FastSpeech & FastSpeech2 & VITS
    speed_control_alpha=1.0,
    # Only for VITS
    noise_scale=0.333,
    noise_scale_dur=0.333,
    use_teacher_forcing=True,
)


import glob
import os
import numpy as np
import kaldiio

# Get model directory path
from espnet_model_zoo.downloader import ModelDownloader
d = ModelDownloader()
model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])

# X-vector selection
spembs = None
if text2speech.use_spembs:
    xvector_ark = [p for p in glob.glob(f"{model_dir}/../../dump/**/spk_xvector.ark", recursive=True) if "tr" in p][0]
    xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
    spks = list(xvectors.keys())

    # randomly select speaker
    random_spk_idx = np.random.randint(0, len(spks))
    spk = spks[random_spk_idx]
    spembs = xvectors[spk]
    print(f"selected spk: {spk}")

# Speaker ID selection
sids = None
if text2speech.use_sids:
    spk2sid = glob.glob(f"{model_dir}/../../dump/**/spk2sid", recursive=True)[0]
    with open(spk2sid) as f:
        lines = [line.strip() for line in f.readlines()]
    sid2spk = {int(line.split()[1]): line.split()[0] for line in lines}
    
    # randomly select speaker
    sids = np.array(np.random.randint(1, len(sid2spk)))
    spk = sid2spk[int(sids)]
    print(f"selected spkid: {spk}")

# Reference speech selection for GST
speech = None
if text2speech.use_speech:
    # you can change here to load your own reference speech
    # e.g.
     import soundfile as sf
     speech, fs = sf.read("/home/chitralekha/espnet/egs/timit/asr2/M09_B1_C1_M2.wav")
     speech = torch.from_numpy(speech).float()
     print(f"selected wav")
    #speech = torch.randn(50000,) * 0.01
    

wav = text2speech("foobar",speech=speech, spembs=spembs, sids=sids)["wav"].cpu()
soundfile.write("out.wav", wav.numpy(), text2speech.fs, "PCM_16")

Output:
selected spkid: p341
selected wav
Traceback (most recent call last):
  File "vc_multispeaker.py", line 82, in <module>
    wav = text2speech("foobar",speech=speech, spembs=spembs, sids=sids)["wav"].cpu()
  File "/home/chitralekha/espent_latest/espnet/tools/venv/lib/python3.7/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
    return func(*args, **kwargs)
  File "/home/chitralekha/espent_latest/espnet/espnet2/bin/tts_inference.py", line 196, in __call__
    output_dict = self.model.inference(**batch, **cfg)
  File "/home/chitralekha/espent_latest/espnet/espnet2/tts/espnet_model.py", line 299, in inference
    output_dict = self.tts.inference(**input_dict, **decode_config)
TypeError: inference() got an unexpected keyword argument 'speech'
```



Discussion:
Teacher forcing requires ground-truth speech, so the error is happened.
In other words, you cannot use teacher forcing option without ground truth speech.
Do you mean the speech on which x-vectors were trained?

Sorry, I misunderstood your question.
Before calling `inference`, could you try to set `text2speech.tts.require_raw_waveform = False` ?
(Following lines should be skipped)
https://github.com/espnet/espnet/blob/a4c5547b1b484141e79cee6d972c41247c1cdaa3/espnet2/tts/espnet_model.py#L263-L264

i tried setting the attribute text2speech.tts.require_raw_waveform = False
and put a print statement inside espent_model.py as
print(getattr(self.tts,"require_raw_speech"))
        if decode_config["use_teacher_forcing"] or getattr(self.tts, "use_gst", False):
            if speech is None:
                raise RuntimeError("missing required argument: 'speech'")
            if self.feats_extract is not None:
                feats = self.feats_extract(speech[None])[0][0]
            else:
                # Use precalculated feats (feats_type != raw case)
                feats = speech
            if self.normalize is not None:
                feats = self.normalize(feats[None])[0][0]
            input_dict.update(feats=feats)
            
            if self.tts.require_raw_speech:
                input_dict.update(speech=speech)
                print("self.tts.require_raw_speech")

The output is 
True
self.tts.require_raw_speech

If I try to set text2speech.tts.require_raw_speech = False, 
I get the below error:
   text2speech.tts.require_raw_speech = False
  File "/home/chitralekha/espent_latest/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1317, in __setattr__
    super().__setattr__(name, value)
AttributeError: can't set attribute


Sorry for my typo. I mean `require_raw_speech`.
How about simply comment out the following lines?
https://github.com/espnet/espnet/blob/a4c5547b1b484141e79cee6d972c41247c1cdaa3/espnet2/tts/espnet_model.py#L263-L264
Teacher forcing requires ground-truth speech, so the error is happened.
In other words, you cannot use teacher forcing option without ground truth speech.
Do you mean the speech on which x-vectors were trained?

Sorry, I misunderstood your question.
Before calling `inference`, could you try to set `text2speech.tts.require_raw_waveform = False` ?
(Following lines should be skipped)
https://github.com/espnet/espnet/blob/a4c5547b1b484141e79cee6d972c41247c1cdaa3/espnet2/tts/espnet_model.py#L263-L264

i tried setting the attribute text2speech.tts.require_raw_waveform = False
and put a print statement inside espent_model.py as
print(getattr(self.tts,"require_raw_speech"))
        if decode_config["use_teacher_forcing"] or getattr(self.tts, "use_gst", False):
            if speech is None:
                raise RuntimeError("missing required argument: 'speech'")
            if self.feats_extract is not None:
                feats = self.feats_extract(speech[None])[0][0]
            else:
                # Use precalculated feats (feats_type != raw case)
                feats = speech
            if self.normalize is not None:
                feats = self.normalize(feats[None])[0][0]
            input_dict.update(feats=feats)
            
            if self.tts.require_raw_speech:
                input_dict.update(speech=speech)
                print("self.tts.require_raw_speech")

The output is 
True
self.tts.require_raw_speech

If I try to set text2speech.tts.require_raw_speech = False, 
I get the below error:
   text2speech.tts.require_raw_speech = False
  File "/home/chitralekha/espent_latest/espnet/tools/venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1317, in __setattr__
    super().__setattr__(name, value)
AttributeError: can't set attribute


Sorry for my typo. I mean `require_raw_speech`.
How about simply comment out the following lines?
https://github.com/espnet/espnet/blob/a4c5547b1b484141e79cee6d972c41247c1cdaa3/espnet2/tts/espnet_model.py#L263-L264

Resolving patch:
diff --git a/espnet2/tts/espnet_model.py b/espnet2/tts/espnet_model.py
--- a/espnet2/tts/espnet_model.py
+++ b/espnet2/tts/espnet_model.py
@@ -260,8 +260,6 @@ def inference(
             if self.normalize is not None:
                 feats = self.normalize(feats[None])[0][0]
             input_dict.update(feats=feats)
-            if self.tts.require_raw_speech:
-                input_dict.update(speech=speech)
 
         if decode_config["use_teacher_forcing"]:
             if durations is not None:


Test patch: