app.py

from espnet2.bin.tts_inference import Text2Speech
import torch
from parallel_wavegan.utils import download_pretrained_model, load_model
import gradio as gr
import os
import subprocess
from zipfile import ZipFile

config_path = "config.yaml"
model_path = "model.pth"

vocoder_tag = "ljspeech_parallel_wavegan.v3"

vocoder = load_model(download_pretrained_model(vocoder_tag)).to("cpu").eval()
vocoder.remove_weight_norm()

url = os.environ.get("PHN_URL")
subprocess.call(["wget", url, "-q"])

with ZipFile("phn.zip", "r") as zip_ref:
    zip_ref.extractall()
subprocess.call(["chmod", "+x", "xml_nlp"])

global_styles = {
    "Style 1": torch.load("style1.pt"),
    "Style 2": torch.load("style2.pt"),
    "Style 3": torch.load("style3.pt"),
    "Style 4": torch.load("style4.pt"),
    "Style 5": torch.load("style5.pt"),
    "Style 6": torch.load("style6.pt"),
}


def phonemize(text):
    with open("input.txt", "w+") as f:
        f.write(text)

    with open("input.xml", "w") as f:
        pass

    subprocess.call(["./xml_nlp", "input", "180", "en_us/enu.ini", "en_us"])

    phoneme_string = ""
    with open("input.xml", "r") as f:
        for line in f.readlines():
            phoneme_string += line.split("[")[-1][:-2]
    return phoneme_string


def inference(text, global_style, alpha, prev_fg_inds, input_fg_inds):
    with torch.no_grad():
        text2speech = Text2Speech(
            config_path,
            model_path,
            device="cpu",
            # Only for Tacotron 2
            threshold=0.5,
            minlenratio=0.0,
            maxlenratio=10.0,
            use_att_constraint=False,
            backward_window=1,
            forward_window=3,
            # Only for FastSpeech & FastSpeech2
            speed_control_alpha=alpha,
        )
        text2speech.spc2wav = None  # Disable griffin-lim

        style_emb = torch.flatten(global_styles[global_style])

        phoneme_string = phonemize(text)
        phonemes = phoneme_string.split(" ")

        max_edit_index = -1
        for i in range(len(input_fg_inds) - 1, -1, -1):
            if input_fg_inds[i] != "":
                max_edit_index = i
                break

        if max_edit_index == -1:
            _, c, _, _, _, _, _, output_fg_inds = text2speech(
                phoneme_string, ref_embs=style_emb
            )

        else:
            input_fg_inds_int_list = []
            for i in range(max_edit_index + 1):
                if input_fg_inds[i] != "":
                    input_fg_inds_int_list.append(int(input_fg_inds[i]))
                else:
                    input_fg_inds_int_list.append(prev_fg_inds[i][1])
            input_fg_inds = input_fg_inds_int_list

            prev_fg_inds_list = [[[row[1], row[2], row[3]] for row in prev_fg_inds]]
            prev_fg_inds = torch.tensor(prev_fg_inds_list, dtype=torch.int64)

            fg_inds = torch.tensor(input_fg_inds_int_list).unsqueeze(0)
            _, c, _, _, _, _, _, part_output_fg_inds = text2speech(
                phoneme_string, ref_embs=style_emb, fg_inds=fg_inds
            )

            prev_fg_inds[0, max_edit_index + 1 :, :] = part_output_fg_inds[0]
            output_fg_inds = prev_fg_inds

        output_fg_inds_list = output_fg_inds.tolist()[0]
        padded_phonemes = ["", *phonemes]
        dataframe_values = [
            [phoneme, *fgs]
            for phoneme, fgs in zip(padded_phonemes, output_fg_inds_list)
        ]
        selected_inds = [
            [input_fg_inds[i]] if i < len(input_fg_inds) else [""]
            for i in range(len(padded_phonemes))
        ]
        wav = vocoder.inference(c)

        return [
            (22050, wav.view(-1).cpu().numpy()),
            dataframe_values,
            selected_inds,
        ]


demo = gr.Blocks()

with demo:
    gr.Markdown(
        """
    
    # ConEx Demo
    
    This demo shows the capabilities of ConEx, a model for **Con**trollable **Ex**pressive speech synthesis. 
    ConEx allows you to generate speech in a certain speaking style, and gives you the ability to edit the prosody* of the generated speech at a fine level.
    We proposed ConEx in our paper titled ["Interactive Multi-Level Prosody Control for Expressive Speech Synthesis"](https://jessa.github.io/assets/pdf/cornille2022icassp.pdf), published in proceedings of the IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP) 2022.

    To convert text to speech: input some text, choose the desired speaking style, set the duration factor (higher = slower speech), and press "Generate speech".
    
    **prosody refers to speech characteristics such as intonation, stress, rhythm*
    """
    )

    with gr.Row():
        text_input = gr.Textbox(
            label="Input text",
            lines=4,
            placeholder="E.g. I didn't say he stole the money",
        )

        with gr.Column():
            global_style_dropdown = gr.Dropdown(
                ["Style 1", "Style 2", "Style 3", "Style 4", "Style 5", "Style 6"],
                value="Style 1",
                label="Global speaking style",
            )
            alpha_slider = gr.Slider(
                0.1, 2, value=1, step=0.1, label="Alpha (duration factor)"
            )

    audio = gr.Audio()
    with gr.Row():
        button = gr.Button("Generate Speech")

    gr.Markdown(
        """
    
    ### Fine-grained prosody editor
    Once you've generated some speech, the following table will show the id of the prosody embedding used for each phoneme.
    A prosody embedding determines the prosody of the phoneme. 
    The table not only shows the prosody embeddings that are used by default (the top predictions), but also two more likely prosody embeddings.
    
    In order to change the prosody of a phoneme, write a new prosody embedding id in the "Chosen prosody embeddings" column and press "Generate speech" again. 
    You can use any number from 0-31, but the 2nd and 3rd predictions are more likely to give a fitting prosody.
    Based on your edit, new prosody embeddings will be generated for the phonemes after the edit. 
    Thus, you can iteratively change the prosody by starting from the beginning of the utterance and working your through the utterance, making edits as you see fit.
    The prosody embeddings before your edit will remain the same as before, and will be copied to the "Chosen prosody embeddings" column.
    """
    )

    with gr.Row():
        phoneme_preds_df = gr.Dataframe(
            headers=["Phoneme", "🥇 Top pred", "🥈 2nd pred", "🥉 3rd pred"],
            type="array",
            col_count=(4, "static"),
        )
        phoneme_edits_df = gr.Dataframe(
            headers=["Chosen prosody embeddings"], type="array", col_count=(1, "static")
        )

    button.click(
        inference,
        inputs=[
            text_input,
            global_style_dropdown,
            alpha_slider,
            phoneme_preds_df,
            phoneme_edits_df,
        ],
        outputs=[audio, phoneme_preds_df, phoneme_edits_df],
    )


demo.launch()