import gradio as gr import librosa import numpy as np import torch import torch.nn.functional as F from pathlib import Path from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5ForSpeechToSpeech, SpeechT5HifiGan from speechbrain.pretrained import EncoderClassifier title = "SpeechT5: Voice Conversion" description = """ The SpeechT5 model is pre-trained on text as well as speech inputs, with targets that are also a mix of text and speech. By pre-training on text and speech at the same time, it learns unified representations for both, resulting in improved modeling capabilities. SpeechT5 can be fine-tuned for different speech tasks. This space demonstrates the text-to-speech and speech-to-speech checkpoints for (American) English language voice cloning.

How to use: Upload target voice audio file or select from the list [or record using the microphone -TBD]. The audio is converted to mono and resampled to 16 kHz before being passed into the EncoderClassifier model to obtain target voice embedding. Enter text in the text box or upload source voice audio file [or record using the microphone -TBD]. The output is a mel spectrogram, which is converted to a mono 16 kHz waveform by the HiFi-GAN vocoder. Because the model always applies random dropout, each attempt will give slightly different results. """ article = """

Original demos: Speech recognition (ASR) demo | TTS demo | Voice Conversion demo | An interactive Colab notebook | Fine-tunining SpeechT5 TTS

References: SpeechT5 paper | SpeechT5 blog post | original GitHub | original weights

@article{Ao2021SpeechT5,
  title   = {SpeechT5: Unified-Modal Encoder-Decoder Pre-training for Spoken Language Processing},
  author  = {Junyi Ao and Rui Wang and Long Zhou and Chengyi Wang and Shuo Ren and Yu Wu and Shujie Liu and Tom Ko and Qing Li and Yu Zhang and Zhihua Wei and Yao Qian and Jinyu Li and Furu Wei},
  eprint={2110.07205},
  archivePrefix={arXiv},
  primaryClass={eess.AS},
  year={2021}
}

Speaker embeddings were generated from CMU ARCTIC using this script.

""" device = "cuda" if torch.cuda.is_available() else "cpu" checkpoint = "microsoft/speecht5_vc" processor_vc = SpeechT5Processor.from_pretrained(checkpoint) model_vc = SpeechT5ForSpeechToSpeech.from_pretrained(checkpoint) checkpoint_tts = "microsoft/speecht5_tts" processor_tts = SpeechT5Processor.from_pretrained(checkpoint_tts) model_tts = SpeechT5ForTextToSpeech.from_pretrained(checkpoint_tts) vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan") model_embed = { "speechbrain/spkrec-xvect-voxceleb": 512, "speechbrain/spkrec-ecapa-voxceleb": 192, } checkpoint_embed = "speechbrain/spkrec-xvect-voxceleb" size_embed = model_embed[checkpoint_embed] embeding_classifier = EncoderClassifier.from_hparams(source=checkpoint_embed, run_opts={"device": device}, savedir="/tmp/speaker_embed") examples_pt = 'examples' allowed_extentions = ['.mp3', '.wav'] examples = {f.name: f for f in Path(examples_pt).glob('*') if f.suffix in allowed_extentions} default_voice = list(examples.keys())[0] verse = """Mary had a little lamb, Its fleece was white as snow. Everywhere the child went, The little lamb was sure to go.""" def process_audio(sampling_rate, waveform, target_sr=16000): # convert from int16 to floating point waveform = waveform / 32678.0 # convert to mono if stereo if len(waveform.shape) > 1: waveform = librosa.to_mono(waveform.T) # resample to 16 kHz if necessary if sampling_rate != target_sr: waveform = librosa.resample(waveform, orig_sr=sampling_rate, target_sr=target_sr) # limit to 30 seconds waveform = waveform[:target_sr * 30] # make PyTorch tensor waveform = torch.tensor(waveform) return waveform def f2embed(waveform, sz): with torch.no_grad(): embeddings = embeding_classifier.encode_batch(waveform) embeddings = F.normalize(embeddings, dim=2) embeddings = embeddings.squeeze().cpu().numpy() assert embeddings.shape[0] == sz, embeddings.shape[0] return embeddings def on_voicedropdown(x): return examples[x] def on_voiceload(audio, sz=size_embed): print("on_voiceload") # audio = tuple (sample_rate, frames) or (sample_rate, (frames, channels)) if audio is not None: sampling_rate, waveform = audio else: return np.zeros(sz) waveform = process_audio(sampling_rate, waveform) embed = f2embed(waveform, sz) print("Generated embedding", embed[:5]) return embed def voice_clone(audio, speaker_embedding, target_sr=16000): # audio = tuple (sample_rate, frames) or (sample_rate, (frames, channels)) if audio is None or speaker_embedding is None: return (target_sr, np.zeros(0).astype(np.int16)) else: sampling_rate, waveform = audio waveform = process_audio(sampling_rate, waveform) inputs = processor_vc(audio=waveform, sampling_rate=target_sr, return_tensors="pt") speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0) speech = model_vc.generate_speech(inputs["input_values"], speaker_embedding, vocoder=vocoder) speech = (speech.numpy() * 32767).astype(np.int16) return (target_sr, speech) def text_to_speech(text, speaker_embedding, target_sr=16000): if len(text.strip()) == 0 or speaker_embedding is None: return (target_sr, np.zeros(0).astype(np.int16)) inputs = processor_tts(text=text, return_tensors="pt") # limit input length input_ids = inputs["input_ids"] input_ids = input_ids[..., :model_tts.config.max_text_positions] speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0) speech = model_tts.generate_speech(input_ids, speaker_embedding, vocoder=vocoder) speech = (speech.numpy() * 32767).astype(np.int16) return (target_sr, speech) theme = gr.themes.Monochrome() with gr.Blocks() as demo: voice_embedding = gr.State(None) def activate(*args): return gr.update(interactive=True) if len(args) == 1 else [gr.update(interactive=True)] * len(args) def deactivate(*args): return gr.update(interactive=False) if len(args) == 1 else [gr.update(interactive=False)] * len(args) gr.Markdown(description) with gr.Accordion("Voice to clone", open=False) as accordion: gr.Markdown("Upload target voice...") with gr.Row(equal_height=True): voice_upload = gr.Audio(label="Upload target voice", source="upload", type="numpy") voice_dropdown = gr.Dropdown(examples, label='Examples', interactive=True) # TODO: couldn't catch microphone stop event # mic = gr.Audio(label="Record Speech", source="microphone", type="numpy") # mic.stop(fn=lambda x: print('mic stop'), inputs=None, outputs=None) with gr.Row(equal_height=True): with gr.Column(scale=2): with gr.Row(equal_height=True): text_to_convert = gr.Textbox(verse) voice_to_convert = gr.Audio(label="Upload voice to convert", source="upload", type="numpy") with gr.Row(equal_height=True): button_text = gr.Button("Text to speech", interactive=False) button_audio = gr.Button("Convert audio", interactive=False) with gr.Row(equal_height=True): speech = gr.Audio(label="Converted Speech", type="numpy", visible=True, interactive=False) # actions kwargs = dict(fn=on_voiceload, inputs=voice_upload, outputs=voice_embedding) voice_upload.upload(deactivate, [button_text, button_audio], [button_text, button_audio]).\ then(**kwargs).then(activate, [button_text, button_audio], [button_text, button_audio]) voice_dropdown.change(deactivate, [button_text, button_audio], [button_text, button_audio]).\ then(fn=on_voicedropdown, inputs=voice_dropdown, outputs=voice_upload).\ then(**kwargs).then(activate, [button_text, button_audio], [button_text, button_audio]) button_text.click(deactivate, [button_text, button_audio], [button_text, button_audio]).\ then(fn=text_to_speech, inputs=[text_to_convert, voice_embedding], outputs=speech).\ then(activate, [button_text, button_audio], [button_text, button_audio]) button_audio.click(deactivate, [button_text, button_audio], [button_text, button_audio]).\ then(fn=voice_clone, inputs=[voice_to_convert, voice_embedding], outputs=speech).\ then(activate, [button_text, button_audio], [button_text, button_audio]) gr.HTML(article) demo.launch(share=False)