add refer wav support

app.py

@@ -5,6 +5,7 @@ import utils
 from models import SynthesizerTrn
 from text.symbols import symbols
 from text import text_to_sequence
+from emotion_extract import extract_wav
 import numpy as np
 
 
@@ -32,13 +33,13 @@ emotion_dict = {
     "平静2": 3554
 }
 import random
-def tts(txt, emotion):
+def tts(txt, emotion, sid=0):
     stn_tst = get_text(txt, hps)
     randsample = None
     with torch.no_grad():
         x_tst = stn_tst.unsqueeze(0)
         x_tst_lengths = torch.LongTensor([stn_tst.size(0)])
-        sid = torch.LongTensor([0])
+        sid = torch.LongTensor([sid])
         if type(emotion) ==int:
             emo = torch.FloatTensor(all_emotions[emotion]).unsqueeze(0)
         elif emotion == "random":
@@ -57,54 +58,69 @@ def tts(txt, emotion):
     return audio, randsample
 
 
-def tts1(text, emotion):
+def tts1(text, emotion, sid=0):
     if len(text) > 150:
         return "Error: Text is too long", None
-    audio, _ = tts(text, emotion)
+    audio, _ = tts(text, emotion, sid)
     return "Success", (hps.data.sampling_rate, audio)
 
-def tts2(text):
+def tts2(text, sid=0):
     if len(text) > 150:
         return "Error: Text is too long", None
-    audio, randsample = tts(text, "random_sample")
+    audio, randsample = tts(text, "random_sample", sid)
 
     return str(randsample), (hps.data.sampling_rate, audio)
 
-def tts3(text, sample):
+def tts3(text, sample, sid=0):
     if len(text) > 150:
         return "Error: Text is too long", None
     try:
-        audio, _ = tts(text, int(sample))
+        audio, _ = tts(text, int(sample), sid)
         return "Success", (hps.data.sampling_rate, audio)
     except:
         return "输入参数不为整数或其他错误", None
+
+
+def tts4(refer_wav_path, text, sid=0):
+    audio, _ = tts(text, refer_wav_path, sid)
+    return "Success", (hps.data.sampling_rate, audio)
+
+
 app = gr.Blocks()
 with app:
     with gr.Tabs():
         with gr.TabItem("使用预制情感合成"):
+            tts_spk_id = gr.Dropdown(label="speaker", choices=list(range(hps.data.n_speakers)), value=0)
             tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
-            tts_input2 = gr.Dropdown(label="情感", choices=list(emotion_dict.keys()),  value="平静1")
+            tts_input2 = gr.Dropdown(label="情感", choices=list(emotion_dict.keys()), value="平静1")
             tts_submit = gr.Button("合成音频", variant="primary")
             tts_output1 = gr.Textbox(label="Message")
             tts_output2 = gr.Audio(label="Output")
-            tts_submit.click(tts1, [tts_input1, tts_input2], [tts_output1, tts_output2])
+            tts_submit.click(tts1, [tts_input1, tts_input2, tts_spk_id], [tts_output1, tts_output2])
         with gr.TabItem("随机抽取训练集样本作为情感参数"):
+            tts_spk_id = gr.Dropdown(label="speaker", choices=list(range(hps.data.n_speakers)), value=0)
             tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
             tts_submit = gr.Button("合成音频", variant="primary")
             tts_output1 = gr.Textbox(label="随机样本id（可用于第三个tab中合成）")
             tts_output2 = gr.Audio(label="Output")
-            tts_submit.click(tts2, [tts_input1], [tts_output1, tts_output2])
+            tts_submit.click(tts2, [tts_input1, tts_spk_id], [tts_output1, tts_output2])
 
         with gr.TabItem("使用情感样本id作为情感参数"):
-
+            tts_spk_id = gr.Dropdown(label="speaker", choices=list(range(hps.data.n_speakers)), value=0)
             tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
             tts_input2 = gr.Number(label="情感样本id", value=2004)
             tts_submit = gr.Button("合成音频", variant="primary")
             tts_output1 = gr.Textbox(label="Message")
             tts_output2 = gr.Audio(label="Output")
-            tts_submit.click(tts3, [tts_input1, tts_input2], [tts_output1, tts_output2])
+            tts_submit.click(tts3, [tts_input1, tts_input2, tts_spk_id], [tts_output1, tts_output2])
 
         with gr.TabItem("使用参考音频作为情感参数"):
-            tts_input1 = gr.TextArea(label="text", value="暂未实现")
+            tts_spk_id = gr.Dropdown(label="speaker", choices=list(range(hps.data.n_speakers)), value=0)
+            tts_refer_wav = gr.File(label="参考音频")
+            tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
+            tts_submit = gr.Button("合成音频", variant="primary")
+            tts_output1 = gr.Textbox(label="Message")
+            tts_output2 = gr.Audio(label="Output")
+            tts_submit.click(tts4, [tts_refer_wav, tts_input1, tts_spk_id], [tts_output1, tts_output2])
 
     app.launch()

emotion_extract.py

@@ -74,6 +74,7 @@ def process_func(
     y = processor(x, sampling_rate=sampling_rate)
     y = y['input_values'][0]
     y = torch.from_numpy(y).to(device)
+    y = y.unsqueeze(0)
 
     # run through model
     with torch.no_grad():
@@ -89,13 +90,13 @@ def process_func(
 #     wav, sr = librosa.load(f"{rootpath}/{wavname}", 16000)
 #     display(ipd.Audio(wav, rate=sr))
 
-rootpath = "dataset/nene"
+rootpath = "dataset"
 embs = []
 wavnames = []
 def extract_dir(path):
     rootpath = path
     for idx, wavname in enumerate(os.listdir(rootpath)):
-        wav, sr =librosa.load(f"{rootpath}/{wavname}", 16000)
+        wav, sr =librosa.load(f"{rootpath}/{wavname}", sr=16000)
         emb = process_func(np.expand_dims(wav, 0), sr, embeddings=True)
         embs.append(emb)
         wavnames.append(wavname)
@@ -103,10 +104,11 @@ def extract_dir(path):
         print(idx, wavname)
 
 def extract_wav(path):
-    wav, sr = librosa.load(path, 16000)
+    wav, sr = librosa.load(path, sr=16000)
     emb = process_func(np.expand_dims(wav, 0), sr, embeddings=True)
     return emb
 
 if __name__ == '__main__':
-    for spk in ["serena", "koni", "nyaru","shanoa", "mana"]:
+    # for spk in ["serena", "koni", "nyaru","shanoa", "mana"]:
+    for spk in ["dubbingx"]:
         extract_dir(f"dataset/{spk}")