|
import gradio as gr |
|
import librosa |
|
import numpy as np |
|
import torch |
|
import torch.nn.functional as F |
|
from pathlib import Path |
|
|
|
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5ForSpeechToSpeech, SpeechT5HifiGan |
|
from speechbrain.pretrained import EncoderClassifier |
|
|
|
title = "SpeechT5: Voice Conversion" |
|
description = """ |
|
The <b>SpeechT5</b> model is pre-trained on text as well as speech inputs, with targets that are also a mix of text and speech. |
|
By pre-training on text and speech at the same time, it learns unified representations for both, resulting in improved modeling capabilities. |
|
SpeechT5 can be fine-tuned for different speech tasks. This space demonstrates the <b>text-to-speech</b> and <b>speech-to-speech</b> checkpoints for (American) English |
|
language voice cloning. |
|
<p><b>How to use:</b> Upload target voice audio file or select from the list [or record using the microphone -TBD]. The audio is converted to mono and resampled to 16 kHz before |
|
being passed into the EncoderClassifier model to obtain target voice embedding. Enter text in the text box or upload source voice audio file [or record using the microphone -TBD]. |
|
The output is a mel spectrogram, which is converted to a mono 16 kHz waveform by the HiFi-GAN vocoder. |
|
Because the model always applies random dropout, each attempt will give slightly different results. |
|
""" |
|
article = """ |
|
<div style='margin:20px auto;'> |
|
<p>Original demos: <a href="https://huggingface.co/spaces/Matthijs/speecht5-asr-demo">Speech recognition (ASR) demo</a> | |
|
<a href="https://huggingface.co/spaces/Matthijs/speecht5-tts-demo">TTS demo</a> | |
|
<a href="https://huggingface.co/spaces/Matthijs/speecht5-vc-demo">Voice Conversion demo</a> | |
|
<a href="https://colab.research.google.com/drive/1XnOnCsmEmA3lHmzlNRNxRMcu80YZQzYf?usp=sharing">An interactive Colab notebook</a> | |
|
<a href="https://colab.research.google.com/drive/1i7I5pzBcU3WDFarDnzweIj4-sVVoIUFJ">Fine-tunining SpeechT5 TTS</a> |
|
|
|
<p>References: <a href="https://arxiv.org/abs/2110.07205">SpeechT5 paper</a> | |
|
<a href="https://huggingface.co/blog/speecht5">SpeechT5 blog post</a> | |
|
<a href="https://github.com/microsoft/SpeechT5/">original GitHub</a> | |
|
<a href="https://huggingface.co/mechanicalsea/speecht5-vc">original weights</a></p> |
|
<pre> |
|
@article{Ao2021SpeechT5, |
|
title = {SpeechT5: Unified-Modal Encoder-Decoder Pre-training for Spoken Language Processing}, |
|
author = {Junyi Ao and Rui Wang and Long Zhou and Chengyi Wang and Shuo Ren and Yu Wu and Shujie Liu and Tom Ko and Qing Li and Yu Zhang and Zhihua Wei and Yao Qian and Jinyu Li and Furu Wei}, |
|
eprint={2110.07205}, |
|
archivePrefix={arXiv}, |
|
primaryClass={eess.AS}, |
|
year={2021} |
|
} |
|
</pre> |
|
<p>Speaker embeddings were generated from <a href="http://www.festvox.org/cmu_arctic/">CMU ARCTIC</a> using <a href="https://huggingface.co/mechanicalsea/speecht5-vc/blob/main/manifest/utils/prep_cmu_arctic_spkemb.py">this script</a>.</p> |
|
</div> |
|
""" |
|
|
|
device = "cuda" if torch.cuda.is_available() else "cpu" |
|
|
|
checkpoint = "microsoft/speecht5_vc" |
|
processor_vc = SpeechT5Processor.from_pretrained(checkpoint) |
|
model_vc = SpeechT5ForSpeechToSpeech.from_pretrained(checkpoint) |
|
checkpoint_tts = "microsoft/speecht5_tts" |
|
processor_tts = SpeechT5Processor.from_pretrained(checkpoint_tts) |
|
model_tts = SpeechT5ForTextToSpeech.from_pretrained(checkpoint_tts) |
|
vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan") |
|
|
|
|
|
model_embed = { |
|
"speechbrain/spkrec-xvect-voxceleb": 512, |
|
"speechbrain/spkrec-ecapa-voxceleb": 192, |
|
} |
|
checkpoint_embed = "speechbrain/spkrec-xvect-voxceleb" |
|
size_embed = model_embed[checkpoint_embed] |
|
embeding_classifier = EncoderClassifier.from_hparams(source=checkpoint_embed, run_opts={"device": device}, savedir="/tmp/speaker_embed") |
|
|
|
examples_pt = 'examples' |
|
allowed_extentions = ['.mp3', '.wav'] |
|
examples = {f.name: f for f in Path(examples_pt).glob('*') if f.suffix in allowed_extentions} |
|
default_voice = list(examples.keys())[0] |
|
verse = """Mary had a little lamb, |
|
Its fleece was white as snow. |
|
Everywhere the child went, |
|
The little lamb was sure to go.""" |
|
|
|
def process_audio(sampling_rate, waveform, target_sr=16000): |
|
|
|
waveform = waveform / 32678.0 |
|
|
|
|
|
if len(waveform.shape) > 1: |
|
waveform = librosa.to_mono(waveform.T) |
|
|
|
|
|
if sampling_rate != target_sr: |
|
waveform = librosa.resample(waveform, orig_sr=sampling_rate, target_sr=target_sr) |
|
|
|
|
|
waveform = waveform[:target_sr * 30] |
|
|
|
|
|
waveform = torch.tensor(waveform) |
|
return waveform |
|
|
|
|
|
def f2embed(waveform, sz): |
|
with torch.no_grad(): |
|
embeddings = embeding_classifier.encode_batch(waveform) |
|
embeddings = F.normalize(embeddings, dim=2) |
|
embeddings = embeddings.squeeze().cpu().numpy() |
|
assert embeddings.shape[0] == sz, embeddings.shape[0] |
|
return embeddings |
|
|
|
|
|
def on_voicedropdown(x): |
|
return examples[x] |
|
|
|
|
|
def on_voiceload(audio, sz=size_embed): |
|
print("on_voiceload") |
|
|
|
if audio is not None: |
|
sampling_rate, waveform = audio |
|
else: |
|
return np.zeros(sz) |
|
waveform = process_audio(sampling_rate, waveform) |
|
embed = f2embed(waveform, sz) |
|
print("Generated embedding", embed[:5]) |
|
return embed |
|
|
|
|
|
def voice_clone(audio, speaker_embedding, target_sr=16000): |
|
|
|
if audio is None or speaker_embedding is None: |
|
return (target_sr, np.zeros(0).astype(np.int16)) |
|
else: |
|
sampling_rate, waveform = audio |
|
|
|
waveform = process_audio(sampling_rate, waveform) |
|
inputs = processor_vc(audio=waveform, sampling_rate=target_sr, return_tensors="pt") |
|
|
|
speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0) |
|
|
|
speech = model_vc.generate_speech(inputs["input_values"], speaker_embedding, vocoder=vocoder) |
|
|
|
speech = (speech.numpy() * 32767).astype(np.int16) |
|
return (target_sr, speech) |
|
|
|
def text_to_speech(text, speaker_embedding, target_sr=16000): |
|
if len(text.strip()) == 0 or speaker_embedding is None: |
|
return (target_sr, np.zeros(0).astype(np.int16)) |
|
|
|
inputs = processor_tts(text=text, return_tensors="pt") |
|
|
|
input_ids = inputs["input_ids"] |
|
input_ids = input_ids[..., :model_tts.config.max_text_positions] |
|
|
|
speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0) |
|
|
|
speech = model_tts.generate_speech(input_ids, speaker_embedding, vocoder=vocoder) |
|
|
|
speech = (speech.numpy() * 32767).astype(np.int16) |
|
return (target_sr, speech) |
|
|
|
theme = gr.themes.Monochrome() |
|
with gr.Blocks() as demo: |
|
voice_embedding = gr.State(None) |
|
def activate(*args): |
|
return gr.update(interactive=True) if len(args) == 1 else [gr.update(interactive=True)] * len(args) |
|
def deactivate(*args): |
|
return gr.update(interactive=False) if len(args) == 1 else [gr.update(interactive=False)] * len(args) |
|
|
|
gr.Markdown(description) |
|
|
|
with gr.Accordion("Voice to clone", open=False) as accordion: |
|
gr.Markdown("Upload target voice...") |
|
with gr.Row(equal_height=True): |
|
voice_upload = gr.Audio(label="Upload target voice", source="upload", type="numpy") |
|
voice_dropdown = gr.Dropdown(examples, label='Examples', interactive=True) |
|
|
|
|
|
|
|
|
|
|
|
with gr.Row(equal_height=True): |
|
with gr.Column(scale=2): |
|
with gr.Row(equal_height=True): |
|
text_to_convert = gr.Textbox(verse) |
|
voice_to_convert = gr.Audio(label="Upload voice to convert", source="upload", type="numpy") |
|
with gr.Row(equal_height=True): |
|
button_text = gr.Button("Text to speech", interactive=False) |
|
button_audio = gr.Button("Convert audio", interactive=False) |
|
with gr.Row(equal_height=True): |
|
speech = gr.Audio(label="Converted Speech", type="numpy", visible=True, interactive=False) |
|
|
|
|
|
kwargs = dict(fn=on_voiceload, inputs=voice_upload, outputs=voice_embedding) |
|
voice_upload.upload(deactivate, [button_text, button_audio], [button_text, button_audio]).\ |
|
then(**kwargs).then(activate, [button_text, button_audio], [button_text, button_audio]) |
|
voice_dropdown.change(deactivate, [button_text, button_audio], [button_text, button_audio]).\ |
|
then(fn=on_voicedropdown, inputs=voice_dropdown, outputs=voice_upload).\ |
|
then(**kwargs).then(activate, [button_text, button_audio], [button_text, button_audio]) |
|
button_text.click(deactivate, [button_text, button_audio], [button_text, button_audio]).\ |
|
then(fn=text_to_speech, inputs=[text_to_convert, voice_embedding], outputs=speech).\ |
|
then(activate, [button_text, button_audio], [button_text, button_audio]) |
|
button_audio.click(deactivate, [button_text, button_audio], [button_text, button_audio]).\ |
|
then(fn=voice_clone, inputs=[voice_to_convert, voice_embedding], outputs=speech).\ |
|
then(activate, [button_text, button_audio], [button_text, button_audio]) |
|
|
|
gr.HTML(article) |
|
demo.launch(share=False) |
