import os import sys import importlib from subprocess import call from pathlib import Path import json import math import matplotlib.pyplot as plt import torch from torch import nn from torch.nn import functional as F from torch.utils.data import DataLoader from scipy.io.wavfile import write import gradio as gr import scipy.io.wavfile import numpy as np from libs.audio import wav2ogg, float2pcm def run_cmd(command): try: # print(command) call(command, shell=True) except KeyboardInterrupt: print("Process interrupted") sys.exit(1) current = os.getcwd() full = current + "/vits/monotonic_align" os.chdir(full) run_cmd("python3 setup.py build_ext --inplace") run_cmd("mv vits/monotonic_align/* ./") run_cmd("rm -rf vits") # run_cmd(f"mv {current}/lfs/*.pth {current}/pretrained/") # run_cmd("apt-get install espeak -y") # run_cmd("gdown 'https://drive.google.com/uc?id=1q86w74Ygw2hNzYP9cWkeClGT5X25PvBT'") os.chdir("../..") from lojban.lojban import lojban2ipa sys.path.insert(0, './vits') import vits.commons as commons import vits.utils as utils from vits.data_utils import TextAudioLoader, TextAudioCollate, TextAudioSpeakerLoader, TextAudioSpeakerCollate from vits.models import SynthesizerTrn from vits.text.symbols import symbols from vits.text import _clean_text from vits.text import cleaners from vits.text.symbols import symbols sys.path.insert(0, './nix_tts_simple') from nix_tts_simple.tts import generate_voice language_id_lookup = { "Lojban": "jbo", "Transcription": "ipa", "English": "en", "German": "de", "Greek": "el", "Spanish": "es", "Finnish": "fi", "Russian": "ru", "Hungarian": "hu", "Dutch": "nl", "French": "fr", 'Polish': "pl", 'Portuguese': "pt", 'Italian': "it", } # def download_pretrained(): # if not all(Path(file).exists() for file in ["pretrained_ljs.pth", "pretrained_vctk.pth"]): # url = "https://drive.google.com/uc?id=1q86w74Ygw2hNzYP9cWkeClGT5X25PvBT" # gdown.download_folder(url, quiet=True, use_cookies=False) # Mappings from symbol to numeric ID and vice versa: _symbol_to_id = {s: i for i, s in enumerate(symbols)} _id_to_symbol = {i: s for i, s in enumerate(symbols)} def text_to_sequence(text, language, cleaner_names, mode): '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text. Args: text: string to convert to a sequence cleaner_names: names of the cleaner functions to run the text through Returns: List of integers corresponding to the symbols in the text ''' sequence = [] if language == 'jbo': clean_text = lojban2ipa(text, mode) elif language == 'ipa': clean_text = text else: clean_text = _clean_text(text, cleaner_names) for symbol in clean_text: symbol_id = _symbol_to_id[symbol] sequence += [symbol_id] return clean_text, sequence def get_text(text, language, hps, mode): ipa_text, text_norm = text_to_sequence( text, language, hps.data.text_cleaners, mode) if hps.data.add_blank: text_norm = commons.intersperse(text_norm, 0) text_norm_tensor = torch.LongTensor(text_norm) return ipa_text, text_norm_tensor def load_checkpoints(): hps = utils.get_hparams_from_file(current + "/vits/configs/ljs_base.json") model = SynthesizerTrn( len(symbols), hps.data.filter_length // 2 + 1, hps.train.segment_size // hps.data.hop_length, **hps.model) _ = model.eval() _ = utils.load_checkpoint(current + "/pretrained/vits/pretrained_ljs.pth", model, None) hps_vctk = utils.get_hparams_from_file(current + "/vits/configs/vctk_base.json") net_g_vctk = SynthesizerTrn( len(symbols), hps_vctk.data.filter_length // 2 + 1, hps_vctk.train.segment_size // hps_vctk.data.hop_length, n_speakers=hps_vctk.data.n_speakers, **hps_vctk.model) _ = model.eval() _ = utils.load_checkpoint(current + "/pretrained/vits/pretrained_vctk.pth", net_g_vctk, None) return model, hps, net_g_vctk, hps_vctk def inference(text, language, noise_scale, noise_scale_w, length_scale, voice, file_format): if len(text.strip())==0: return [] language = language.split()[0] language = language_id_lookup[language] if bool( language_id_lookup[language]) else "jbo" result = [] if voice == 'Nix-Deterministic' and language == 'jbo': result = generate_voice(lojban2ipa(text,'nix'), current+"/pretrained/nix-tts/nix-ljspeech-v0.1") elif voice == 'Nix-Stochastic' and language == 'jbo': result = generate_voice(lojban2ipa(text,'nix'), current+"/pretrained/nix-tts/nix-ljspeech-sdp-v0.1") elif voice == 'LJS': ipa_text, stn_tst = get_text(text, language, hps, mode="VITS") with torch.no_grad(): x_tst = stn_tst.unsqueeze(0) x_tst_lengths = torch.LongTensor([stn_tst.size(0)]) audio = model.infer(x_tst, x_tst_lengths, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale)[0][0, 0].data.float().numpy() result = [ipa_text, (hps_vctk.data.sampling_rate, audio)] else: ipa_text, stn_tst = get_text(text, language, hps_vctk, mode="VITS") with torch.no_grad(): x_tst = stn_tst.unsqueeze(0) x_tst_lengths = torch.LongTensor([stn_tst.size(0)]) sid = torch.LongTensor([voice]) audio = model_vctk.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=noise_scale, noise_scale_w=noise_scale_w, length_scale=length_scale)[0][0, 0].data.cpu().float().numpy() result = [ipa_text, (hps_vctk.data.sampling_rate, audio)] if file_format == 'ogg': result = [result[0], wav2ogg(result[1][1], result[1][0], text, language)] else: result = [result[0], (result[1][0], float2pcm(result[1][1]))] return result # download_pretrained() model, hps, model_vctk, hps_vctk = load_checkpoints() defaults = { "text": "coi munje", "language": "Lojban", "noise_scale": .667, "noise_scale_w": .8, "speed": 1.8, "voice": "LJS", "example": ["", "Lojban", 0.667, 0.8, 1.8,"LJS","wav"] } inputs = [] outputs = [] css = """ h1 {font-size:200%;} h2 {font-size:120%;} h2 a {color: #0020c5;text-decoration: underline;} p a {text-decoration: underline;} img {display: inline-block;height:32px;} #velsku { text-align: left; margin: 0; /* display: none; */ /* justify-content: baseline; */ /* align-items: flex-start; */ padding: 0; /* height: 28px; */ width: 100%; bottom: 0px; left: 0px; position: fixed; background: white; z-index: 10; } #velsku_sebenji { padding: 0.1rem; display: flex; white-space: nowrap; text-overflow: ellipsis; box-shadow: 0 0 0 1px rgb(56 136 233), 0 0 0 2px rgb(34 87 213), 0 0 0 3px rgb(38 99 224), 0 0 0 4px rgb(25 65 165); margin-top: 3px; } .velsku_pamei { white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } .velsku_pixra { max-height: 20px; margin-right: 0.2rem; } #velsku a { color: #2b79e0; text-decoration: none; } """ def conditionally_hide_widgets(voice): if str(voice).startswith("Nix"): return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False) else: return gr.update(visible=True), gr.update(visible=True), gr.update(visible=True) title = "

la vitci voksa - Lojban text-to-speech

" description = "

VITS & Nix-TTS text-to-speech adapted to Lojban. Join Lojban Discord live chat to discuss further.

" article = "

VITS: Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech | Nix-TTS: Lightweight and End-to-end Text-to-Speech via Module-wise Distillation

" scripts = """ async () => { const script = document.createElement("script"); script.onload = () => console.log("socket.io loaded") ; script.src = "https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.1/socket.io.js"; document.head.appendChild(script); document.addEventListener("DOMContentLoaded", () => { var socket1Chat_connected; var socket1Chat = io("wss://jbotcan.org:9091", { transports: ["polling", "websocket"], }); socket1Chat.on("connect", function () { console.log(socket1Chat); socket1Chat_connected = true; }); socket1Chat.on("connect_error", function () { console.log("1chat connection error"); }); function trimSocketChunk(text) { return text; } socket1Chat.on("sentFrom", function (data) { if (!socket1Chat_connected) return; const i = data.data; const msg = { d: trimSocketChunk(i.chunk), s: i.channelId, w: i.author, }; const velsku = document.getElementById("velsku_sebenji"); velsku.innerHTML = ` [${msg.s}] ${msg.w}: ${msg.d}`; }); socket1Chat.on("history", function (data) { if (!socket1Chat_connected) return; const i = data.slice(-1)[0]; if (!i) return; const msg = { d: trimSocketChunk(i.chunk), s: i.channelId, w: i.author, }; const velsku = document.getElementById("velsku_sebenji"); velsku.innerHTML = ` [${msg.s}] ${msg.w}: ${msg.d}`; }); }); } """ chat = """
Live chat on Discord
""" with gr.Blocks(css=css) as demo: gr.HTML(title) gr.HTML(description) with gr.Row(): with gr.Column(): input_text = gr.Textbox(lines=4, value=defaults["text"], label="Input text", placeholder="add your text, or click one of the examples to load them") langs = gr.Radio([ 'Lojban', 'English', 'Transcription', ], value=defaults["language"], label="Language") voices = gr.Radio(["LJS", 0, 1, 2, 3, 4, "Nix-Deterministic", "Nix-Stochastic"], value=defaults["voice"], label="Voice") noise_scale = gr.Slider(label="Noise scale", minimum=0, maximum=2, step=0.1, value=defaults["noise_scale"]) noise_scale_w = gr.Slider(label="Noise scale W", minimum=0, maximum=2, step=0.1, value=defaults["noise_scale_w"]) slowness = gr.Slider(label="Slowness", minimum=0.1, maximum=3, step=0.1, value=defaults["speed"]) file_format = gr.Radio(["wav", "ogg"], value="wav", label="File format") inputs = [input_text, langs, noise_scale, noise_scale_w, slowness, voices, file_format] # events vits_inputs = [noise_scale, noise_scale_w, slowness] voices.change(fn=conditionally_hide_widgets, inputs=voices,outputs=vits_inputs) with gr.Column(): ipa_block = gr.Textbox(label="International Phonetic Alphabet") audio = gr.Audio(type="numpy", label="Output audio") outputs = [ ipa_block, audio ] btn = gr.Button("Vocalize") btn.click(fn=inference, inputs=inputs, outputs=outputs, api_name="cupra") examples = list(map(lambda el: el[0:len(el)] + defaults["example"][len(el):], [ ["coi ro do ma nuzba", "Lojban"], ["mi djica lo nu do zvati ti", "Lojban", 0.667, 0.8, 1.8,4], ["mu xagji sofybakni cu zvati le purdi", "Lojban", 0.667, 0.8, 1.8, "Nix-Deterministic"], ["ni'o le pa tirxu be me'e zo .teris. pu ki kansa le za'u pendo be le nei le ka xabju le foldi be loi spati", "Lojban"], [", miː dʒˈiːʃaː loːnʊuː doː zvˈaːtiː tiː.", "Transcription"], ["We propose VITS, Conditional Variational Autoencoder with Adversarial Learning for End-to-End Text-to-Speech.", "English"], ])) gr.Examples(examples, inputs, fn=inference, outputs=outputs, cache_examples=True, run_on_click=True) gr.HTML(article) gr.HTML(chat) # gr.HTML(scripts) demo.launch()