File size: 13,766 Bytes
a956529
19a8f04
 
 
 
08a99f7
 
19a8f04
 
210468c
19a8f04
210468c
 
19a8f04
 
 
18dbf2d
19a8f04
210468c
19a8f04
eac9d77
887727d
 
19a8f04
 
 
 
 
 
 
 
 
 
 
 
 
 
887727d
19a8f04
 
 
18dbf2d
 
 
 
 
 
 
19a8f04
887727d
dffc7d4
18dbf2d
 
 
 
 
 
 
 
 
 
 
 
 
 
d1074fe
 
 
18dbf2d
 
 
 
 
 
27e967e
18dbf2d
 
 
 
 
 
 
 
 
 
d83b987
18dbf2d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27e967e
e7373fa
 
210468c
 
e7373fa
210468c
 
18dbf2d
 
 
 
d82c773
18dbf2d
210468c
19a8f04
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18dbf2d
 
 
 
 
8f014b7
18dbf2d
 
e2f2ce0
18dbf2d
 
 
 
 
 
 
 
 
8f014b7
18dbf2d
 
e2f2ce0
18dbf2d
 
 
 
 
 
 
 
27e967e
74242f2
 
 
18dbf2d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27e967e
 
18dbf2d
27e967e
18dbf2d
b89ec7e
 
27e967e
18dbf2d
 
 
 
e2f2ce0
18dbf2d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6edba4c
18dbf2d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74242f2
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
import argparse
import json
import os
import re
import tempfile
import logging
logging.getLogger('numba').setLevel(logging.WARNING)
import librosa
import numpy as np
import torch
from torch import no_grad, LongTensor
import commons
import utils
import gradio as gr
import gradio.utils as gr_utils
import gradio.processing_utils as gr_processing_utils
from ONNXVITS_infer import SynthesizerTrn
from text import text_to_sequence, _clean_text
from text.symbols import symbols
from mel_processing import spectrogram_torch
import psutil
from datetime import datetime

def audio_postprocess(self, y):
    if y is None:
        return None

    if gr_utils.validate_url(y):
        file = gr_processing_utils.download_to_file(y, dir=self.temp_dir)
    elif isinstance(y, tuple):
        sample_rate, data = y
        file = tempfile.NamedTemporaryFile(
            suffix=".wav", dir=self.temp_dir, delete=False
        )
        gr_processing_utils.audio_to_file(sample_rate, data, file.name)
    else:
        file = gr_processing_utils.create_tmp_copy_of_file(y, dir=self.temp_dir)

    return gr_processing_utils.encode_url_or_file_to_base64(file.name)


language_marks = {
    "日本語": "[JA]",
    "简体中文": "[ZH]",
    "English": "[EN]",
    "Mix": "",
}

gr.Audio.postprocess = audio_postprocess

limitation = os.getenv("SYSTEM") == "spaces"  # limit text and audio length in huggingface spaces
def create_tts_fn(model, hps, speaker_ids):
    def tts_fn(text, speaker, language, speed, is_symbol):
        if limitation:
            text_len = len(re.sub("\[([A-Z]{2})\]", "", text))
            max_len = 150
            if is_symbol:
                max_len *= 3
            if text_len > max_len:
                return "Error: Text is too long", None
        if language is not None:
            text = language_marks[language] + text + language_marks[language]
        speaker_id = speaker_ids[speaker]
        stn_tst = get_text(text, hps, is_symbol)
        with no_grad():
            x_tst = stn_tst.unsqueeze(0)
            x_tst_lengths = LongTensor([stn_tst.size(0)])
            sid = LongTensor([speaker_id])
            audio = model.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=.667, noise_scale_w=0.8,
                                length_scale=1.0 / speed)[0][0, 0].data.cpu().float().numpy()
        del stn_tst, x_tst, x_tst_lengths, sid
        return "Success", (hps.data.sampling_rate, audio)

    return tts_fn

def create_vc_fn(model, hps, speaker_ids):
    def vc_fn(original_speaker, target_speaker, input_audio):
        if input_audio is None:
            return "You need to upload an audio", None
        sampling_rate, audio = input_audio
        duration = audio.shape[0] / sampling_rate
        if limitation and duration > 30:
            return "Error: Audio is too long", None
        original_speaker_id = speaker_ids[original_speaker]
        target_speaker_id = speaker_ids[target_speaker]

        audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
        if len(audio.shape) > 1:
            audio = librosa.to_mono(audio.transpose(1, 0))
        if sampling_rate != hps.data.sampling_rate:
            audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=hps.data.sampling_rate)
        with no_grad():
            y = torch.FloatTensor(audio)
            y = y.unsqueeze(0)
            spec = spectrogram_torch(y, hps.data.filter_length,
                                     hps.data.sampling_rate, hps.data.hop_length, hps.data.win_length,
                                     center=False).to(device)
            spec_lengths = LongTensor([spec.size(-1)]).to(device)
            sid_src = LongTensor([original_speaker_id]).to(device)
            sid_tgt = LongTensor([target_speaker_id]).to(device)
            audio = model.voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt)[0][
                0, 0].data.cpu().float().numpy()
        del y, spec, spec_lengths, sid_src, sid_tgt
        return "Success", (hps.data.sampling_rate, audio)

    return vc_fn

def get_text(text, hps, is_symbol):
    text_norm = text_to_sequence(text, hps.symbols, [] if is_symbol else hps.data.text_cleaners)
    if hps.data.add_blank:
        text_norm = commons.intersperse(text_norm, 0)
    text_norm = LongTensor(text_norm)
    return text_norm

def create_to_symbol_fn(hps):
    def to_symbol_fn(is_symbol_input, input_text, temp_text):
        return (_clean_text(input_text, hps.data.text_cleaners), input_text) if is_symbol_input \
            else (temp_text, temp_text)

    return to_symbol_fn

download_audio_js = """
() =>{{
    let root = document.querySelector("body > gradio-app");
    if (root.shadowRoot != null)
        root = root.shadowRoot;
    let audio = root.querySelector("#{audio_id}").querySelector("audio");
    if (audio == undefined)
        return;
    audio = audio.src;
    let oA = document.createElement("a");
    oA.download = Math.floor(Math.random()*100000000)+'.wav';
    oA.href = audio;
    document.body.appendChild(oA);
    oA.click();
    oA.remove();
}}
"""

models_tts = []
models_vc = []
models_info = [
    {
        "title": "Japanese",
        "languages": ["日本語"],
        "description": "",
        "model_path": "./pretrained_models/G_1153000.pth",
        "config_path": "./configs/uma87.json",
        "examples": [['お疲れ様です,トレーナーさん。', 'Silence Suzuka', '日本語', 1, False],
                        ['張り切っていこう!', 'Kitasan Black', '日本語', 1, False],
                        ['何でこんなに慣れでんのよ,私のほが先に好きだっだのに。', 'Grass Wonder', '日本語', 1, False],
                        ['授業中に出しだら,学校生活終わるですわ。', 'Mejiro Mcqueen', '日本語', 1, False],
                        ['お帰りなさい,お兄様!', 'Rice Shower', '日本語', 1, False],
                        ['私の処女をもらっでください!', 'Rice Shower', '日本語', 1, False]]
    },
    {
        "title": "Japanese",
        "languages": ['日本語', '简体中文', 'English', 'Mix'],
        "description": "",
        "model_path": "./pretrained_models/G_1396000.pth",
        "config_path": "./configs/uma_trilingual.json",
        "examples": [['你好,训练员先生,很高兴见到你。', '草上飞 Grass Wonder (Umamusume Pretty Derby)', '简体中文', 1, False],
                        ['To be honest, I have no idea what to say as examples.', '派蒙 Paimon (Genshin Impact)', 'English', 1, False],
                        ['授業中に出しだら,学校生活終わるですわ。', '綾地 寧々 Ayachi Nene (Sanoba Witch)', '日本語', 1, False]]
    }
]



if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--share", action="store_true", default=False, help="share gradio app")
    args = parser.parse_args()
    for info in models_info:
        name = info['title']
        lang = info['languages']
        examples = info['examples']
        config_path = info['config_path']
        model_path = info['model_path']
        hps = utils.get_hparams_from_file(config_path)
        model = SynthesizerTrn(
            len(hps.symbols),
            hps.data.filter_length // 2 + 1,
            hps.train.segment_size // hps.data.hop_length,
            n_speakers=hps.data.n_speakers,
            **hps.model)
        utils.load_checkpoint(model_path, model, None)
        model.eval()
        speaker_ids = hps.speakers
        speakers = list(hps.speakers.keys())
        models_tts.append((name, speakers, lang, example,
                           hps.symbols, create_tts_fn(model, hps, speaker_ids),
                           create_to_symbol_fn(hps)))
        models_vc.append((name, speakers, create_vc_fn(model, hps, speaker_ids)))
    app = gr.Blocks()
    with app:
        gr.Markdown("# English & Chinese & Japanese Anime TTS\n\n"
                    "![visitor badge](https://visitor-badge.glitch.me/badge?page_id=Plachta.VITS-Umamusume-voice-synthesizer)\n\n"
                    "Including Japanese TTS & Trilingual TTS, speakers are all anime characters. 包含一个纯日语TTS和一个中日英三语TTS模型,主要为二次元角色。"
                    "If you have any suggestions or bug reports, feel free to open discussion in [Community](https://huggingface.co/spaces/Plachta/VITS-Umamusume-voice-synthesizer/discussions).\n\n"
                    "若有bug反馈或建议,请在[Community](https://huggingface.co/spaces/Plachta/VITS-Umamusume-voice-synthesizer/discussions)下开启一个新的Discussion。 \n\n"
                    )
        with gr.Tabs():
            with gr.TabItem("TTS"):
                with gr.Tabs():
                    for i, (name, speakers, lang, example, symbols, tts_fn, to_symbol_fn) in enumerate(models_tts):
                        with gr.TabItem(name):
                            with gr.Row():
                                with gr.Column():
                                    textbox = gr.TextArea(label="Text", placeholder="Type your sentence here (Maximum 150 words)", value="こんにちわ。", elem_id=f"tts-input")
                                    with gr.Accordion(label="Phoneme Input", open=False):
                                        temp_text_var = gr.Variable()
                                        symbol_input = gr.Checkbox(value=False, label="Symbol input")
                                        symbol_list = gr.Dataset(label="Symbol list", components=[textbox],
                                                                 samples=[[x] for x in symbols],
                                                                 elem_id=f"symbol-list")
                                        symbol_list_json = gr.Json(value=symbols, visible=False)
                                    symbol_input.change(to_symbol_fn,
                                                        [symbol_input, textbox, temp_text_var],
                                                        [textbox, temp_text_var])
                                    symbol_list.click(None, [symbol_list, symbol_list_json], textbox,
                                                      _js=f"""
                                    (i, symbols, text) => {{
                                        let root = document.querySelector("body > gradio-app");
                                        if (root.shadowRoot != null)
                                            root = root.shadowRoot;
                                        let text_input = root.querySelector("#tts-input").querySelector("textarea");
                                        let startPos = text_input.selectionStart;
                                        let endPos = text_input.selectionEnd;
                                        let oldTxt = text_input.value;
                                        let result = oldTxt.substring(0, startPos) + symbols[i] + oldTxt.substring(endPos);
                                        text_input.value = result;
                                        let x = window.scrollX, y = window.scrollY;
                                        text_input.focus();
                                        text_input.selectionStart = startPos + symbols[i].length;
                                        text_input.selectionEnd = startPos + symbols[i].length;
                                        text_input.blur();
                                        window.scrollTo(x, y);
                    
                                        text = text_input.value;
                                        
                                        return text;
                                    }}""")
                                    # select character
                                    char_dropdown = gr.Dropdown(choices=speakers, value=speakers[0], label='character')
                                    language_dropdown = gr.Dropdown(choices=lang, value=lang[0], label='language')
                                    duration_slider = gr.Slider(minimum=0.1, maximum=5, value=1, step=0.1, label='时长 Duration')
                                with gr.Column():
                                    text_output = gr.Textbox(label="Message")
                                    audio_output = gr.Audio(label="Output Audio", elem_id="tts-audio")
                                    btn = gr.Button("Generate!")
                                    
                                    download = gr.Button("Download Audio")
                                    download.click(None, [], [], _js=download_audio_js.format(audio_id="tts-audio"))
                                    if len(lang) == 1:
                                        btn.click(tts_fn, inputs=[textbox, char_dropdown, None, duration_slider, symbol_input], 
                                              outputs=[text_output, audio_output])
                                    else:
                                        btn.click(tts_fn, inputs=[textbox, char_dropdown, language_dropdown, duration_slider, symbol_input], 
                                              outputs=[text_output, audio_output])
                            gr.Examples(
                                examples=example,
                                inputs=[textbox, char_dropdown, language_dropdown,
                                        duration_slider, symbol_input],
                                outputs=[text_output, audio_output],
                                fn=tts_fn
                            )
    app.queue(concurrency_count=3).launch(show_api=False, share=args.share)