{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Text to Speech Playground" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/homebrew/Caskroom/miniconda/base/envs/llm/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "import os\n", "\n", "import torch\n", "import gradio as gr\n", "from TTS.api import TTS\n", "os.environ[\"COQUI_TOS_AGREED\"] = \"1\"\n", "# os.environ[\"PYTORCH_ENABLE_MPS_FALLBACK\"] = \"1\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from collections import namedtuple\n", "\n", "Voice = namedtuple('voice', ['name', 'neutral','angry'])\n" ] }, { "cell_type": "code", "execution_count": 84, "metadata": {}, "outputs": [], "source": [ "voices = [\n", " Voice('Attenborough', neutral='audio/attenborough/neutral.wav', angry=None),\n", " Voice('Rick', neutral='audio/rick/neutral.wav', angry=None),\n", " Voice('Freeman', neutral='audio/freeman/neutral.wav', angry='audio/freeman/angry.wav'),\n", " Voice('Walken', neutral='audio/walken/neutral.wav', angry=None),\n", " Voice('Darth Wader', neutral='audio/darth/neutral.wav', angry=None),\n", "]" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[voice(name='Attenborough', neutral='audio/attenborough/neutral.mp3', angry=None),\n", " voice(name='Rick', neutral='audio/rick/neutral.mp3', angry=None),\n", " voice(name='Freeman', neutral='audio/freeman/neutral.mp3', angry='audio/freeman/angry.mp3'),\n", " voice(name='Walken', neutral='audio/walken/neutral.mp3', angry=None),\n", " voice(name='Darth Wader', neutral='audio/darth/neutral.mp3', angry=None)]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "voices" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " > tts_models/multilingual/multi-dataset/xtts_v2 is already downloaded.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/homebrew/Caskroom/miniconda/base/envs/llm/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n", " _torch_pytree._register_pytree_node(\n", "/opt/homebrew/Caskroom/miniconda/base/envs/llm/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n", " _torch_pytree._register_pytree_node(\n", "/opt/homebrew/Caskroom/miniconda/base/envs/llm/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n", " _torch_pytree._register_pytree_node(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " > Using model: xtts\n" ] } ], "source": [ "#load model for text to speech\n", "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "# device = \"mps\"\n", "tts_pipelins = TTS(\"tts_models/multilingual/multi-dataset/xtts_v2\").to(device)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "import IPython\n" ] }, { "cell_type": "code", "execution_count": 81, "metadata": {}, "outputs": [], "source": [ "speaker_embedding_cache = {}" ] }, { "cell_type": "code", "execution_count": 82, "metadata": {}, "outputs": [], "source": [ "def compute_speaker_embedding(voice_path: str, config, pipeline, cache):\n", " if voice_path not in cache:\n", " cache[voice_path] = pipeline.synthesizer.tts_model.get_conditioning_latents(\n", " audio_path=voice_path,\n", " gpt_cond_len=config.gpt_cond_len,\n", " gpt_cond_chunk_len=config.gpt_cond_chunk_len,\n", " max_ref_length=config.max_ref_len,\n", " sound_norm_refs=config.sound_norm_refs,\n", " )\n", " return cache[voice_path]" ] }, { "cell_type": "code", "execution_count": 87, "metadata": {}, "outputs": [], "source": [ "out = compute_speaker_embedding(voices[0].neutral, tts_pipelins.synthesizer.tts_config, tts_pipelins, speaker_embedding_cache)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " > Text splitted to sentences.\n", "['Hey Petra, so you are hungry?', 'and you like me to prepare some strawberries for you?', 'do you like strawberries?']\n", " > Processing time: 15.77448582649231\n", " > Real-time factor: 1.7459813091024587\n" ] } ], "source": [ "out = tts_pipelins.tts(\n", " \"Hello, I am Rick, pickle rick, you took a wrong turn and now you're stuck in a parallel universe\",\n", " speaker_wav=\"audio/freeman/neutral.wav\",\n", " language=\"en\",\n", " # file_path=\"out.wav\",\n", ")" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "from typing import List\n", "import time" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "ref_audio_path = \"audio/freeman/neutral.wav\"" ] }, { "cell_type": "code", "execution_count": 53, "metadata": {}, "outputs": [], "source": [ "config.max_ref_len = 360" ] }, { "cell_type": "code", "execution_count": 78, "metadata": {}, "outputs": [], "source": [ "config = tts_pipelins.synthesizer.tts_config\n", "(gpt_cond_latent, speaker_embedding) = tts_pipelins.synthesizer.tts_model.get_conditioning_latents(\n", " audio_path=ref_audio_path,\n", " gpt_cond_len=config.gpt_cond_len,\n", " gpt_cond_chunk_len=config.gpt_cond_chunk_len,\n", " max_ref_length=config.max_ref_len,\n", " sound_norm_refs=config.sound_norm_refs,\n", ")" ] }, { "cell_type": "code", "execution_count": 107, "metadata": {}, "outputs": [], "source": [ "(gpt_cond_latent, speaker_embedding) = compute_speaker_embedding(voices[0].neutral, tts_pipelins.synthesizer.tts_config, tts_pipelins, speaker_embedding_cache)" ] }, { "cell_type": "code", "execution_count": 114, "metadata": {}, "outputs": [], "source": [ "import numpy as np" ] }, { "cell_type": "code", "execution_count": 116, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(205872,)" ] }, "execution_count": 116, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.array(out)" ] }, { "cell_type": "code", "execution_count": 110, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "205872" ] }, "execution_count": 110, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(out)" ] }, { "cell_type": "code", "execution_count": 128, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " > Text splitted to sentences.\n", "['Something is up!']\n", " > Processing time: 2.9515581130981445\n", " > Real-time factor: 1.588292083019672\n" ] } ], "source": [ "out = tts(\n", " tts_pipelins.synthesizer,\n", " \"Something is up!\",\n", " # speaker_wav=ref_audio_path,\n", " language_name=\"en\",\n", " speaker=None,\n", " gpt_cond_latent=gpt_cond_latent,\n", " speaker_embedding=speaker_embedding,\n", " speed=1.1,\n", " # file_path=\"out.wav\",\n", ")" ] }, { "cell_type": "code", "execution_count": 129, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "execution_count": 129, "metadata": {}, "output_type": "execute_result" } ], "source": [ "IPython.display.Audio(out, rate=22050)" ] }, { "cell_type": "code", "execution_count": 66, "metadata": {}, "outputs": [], "source": [ "from TTS.vocoder.utils.generic_utils import interpolate_vocoder_input\n", "\n", "def tts(\n", " self,\n", " text: str = \"\",\n", " language_name: str = \"\",\n", " reference_wav=None,\n", " gpt_cond_latent=None,\n", " speaker_embedding=None,\n", " split_sentences: bool = True,\n", " **kwargs,\n", ") -> List[int]:\n", " \"\"\"🐸 TTS magic. Run all the models and generate speech.\n", "\n", " Args:\n", " text (str): input text.\n", " speaker_name (str, optional): speaker id for multi-speaker models. Defaults to \"\".\n", " language_name (str, optional): language id for multi-language models. Defaults to \"\".\n", " speaker_wav (Union[str, List[str]], optional): path to the speaker wav for voice cloning. Defaults to None.\n", " style_wav ([type], optional): style waveform for GST. Defaults to None.\n", " style_text ([type], optional): transcription of style_wav for Capacitron. Defaults to None.\n", " reference_wav ([type], optional): reference waveform for voice conversion. Defaults to None.\n", " reference_speaker_name ([type], optional): speaker id of reference waveform. Defaults to None.\n", " split_sentences (bool, optional): split the input text into sentences. Defaults to True.\n", " **kwargs: additional arguments to pass to the TTS model.\n", " Returns:\n", " List[int]: [description]\n", " \"\"\"\n", " start_time = time.time()\n", " wavs = []\n", "\n", " if not text and not reference_wav:\n", " raise ValueError(\n", " \"You need to define either `text` (for sythesis) or a `reference_wav` (for voice conversion) to use the Coqui TTS API.\"\n", " )\n", "\n", " if text:\n", " sens = [text]\n", " if split_sentences:\n", " print(\" > Text splitted to sentences.\")\n", " sens = self.split_into_sentences(text)\n", " print(sens)\n", "\n", " if not reference_wav: # not voice conversion\n", " for sen in sens:\n", " outputs = self.tts_model.inference(\n", " sen,\n", " language_name,\n", " gpt_cond_latent,\n", " speaker_embedding,\n", " # GPT inference\n", " temperature=0.75,\n", " length_penalty=1.0,\n", " repetition_penalty=10.0,\n", " top_k=50,\n", " top_p=0.85,\n", " do_sample=True,\n", " **kwargs,\n", " )\n", " waveform = outputs[\"wav\"]\n", " if torch.is_tensor(waveform) and waveform.device != torch.device(\"cpu\") and not use_gl:\n", " waveform = waveform.cpu()\n", " if not use_gl:\n", " waveform = waveform.numpy()\n", " waveform = waveform.squeeze()\n", "\n", " # trim silence\n", " if \"do_trim_silence\" in self.tts_config.audio and self.tts_config.audio[\"do_trim_silence\"]:\n", " waveform = trim_silence(waveform, self.tts_model.ap)\n", "\n", " wavs += list(waveform)\n", " wavs += [0] * 10000\n", "\n", "\n", " # compute stats\n", " process_time = time.time() - start_time\n", " audio_time = len(wavs) / self.tts_config.audio[\"sample_rate\"]\n", " print(f\" > Processing time: {process_time}\")\n", " print(f\" > Real-time factor: {process_time / audio_time}\")\n", " return wavs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "type(tts_pipelins)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "IPython.display.Audio(out, rate=22050)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def text_to_speech(voice, tts):\n", " return voice.neutral" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ " tts.tts_to_file(text= str(quest_processing[0]),\n", " file_path=\"output.wav\",\n", " speaker_wav=f'Audio_Files/{voice}.wav',\n", " language=quest_processing[3],\n", " emotion = \"angry\")\n", "\n", " audio_path = \"output.wav\"\n", " return audio_path, state['context'], state" ] }, { "cell_type": "code", "execution_count": 90, "metadata": {}, "outputs": [], "source": [ "voice_options = []\n", "for voice in voices:\n", " if voice.neutral:\n", " voice_options.append(f\"{voice.name} - Neutral\")\n", " if voice.angry:\n", " voice_options.append(f\"{voice.name} - Angry\")" ] }, { "cell_type": "code", "execution_count": 101, "metadata": {}, "outputs": [], "source": [ "def voice_from_text(voice):\n", " for v in voices:\n", " if voice == f\"{v.name} - Neutral\":\n", " return v.neutral\n", " if voice == f\"{v.name} - Angry\":\n", " return v.angry" ] }, { "cell_type": "code", "execution_count": 121, "metadata": {}, "outputs": [], "source": [ "def tts_gradio(text, voice, state):\n", " print(text, voice, state)\n", " voice_path = voice_from_text(voice)\n", " (gpt_cond_latent, speaker_embedding) = compute_speaker_embedding(voice_path, tts_pipelins.synthesizer.tts_config, tts_pipelins, speaker_embedding_cache)\n", " out = tts(\n", " tts_pipelins.synthesizer,\n", " text,\n", " language_name=\"en\",\n", " speaker=None,\n", " gpt_cond_latent=gpt_cond_latent,\n", " speaker_embedding=speaker_embedding,\n", " speed=1.1,\n", " # file_path=\"out.wav\",\n", " )\n", " return (22050, np.array(out)), dict(text=text, voice=voice)" ] }, { "cell_type": "code", "execution_count": 122, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "dict_keys(['audio/attenborough/neutral.wav'])" ] }, "execution_count": 122, "metadata": {}, "output_type": "execute_result" } ], "source": [ "speaker_embedding_cache.keys()" ] }, { "cell_type": "code", "execution_count": 127, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This is going to be fun, let's enjoy ourselves\n", "Closing server running on port: 7860\n", "Closing server running on port: 7860\n", "Closing server running on port: 7860\n", "Closing server running on port: 7860\n", "Closing server running on port: 7860\n", "Closing server running on port: 7860\n", "Closing server running on port: 7860\n", "Closing server running on port: 7860\n", "Closing server running on port: 7860\n", "Closing server running on port: 7860\n", "Closing server running on port: 7860\n", "Running on local URL: http://0.0.0.0:7860\n", "\n", "To create a public link, set `share=True` in `launch()`.\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "This is going to be fun, let's enjoy ourselves Darth Wader - Neutral None\n", " > Text splitted to sentences.\n", "[\"This is going to be fun, let's enjoy ourselves\"]\n", " > Processing time: 9.152068138122559\n", " > Real-time factor: 1.8119083325456329\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/homebrew/Caskroom/miniconda/base/envs/llm/lib/python3.11/site-packages/gradio/processing_utils.py:390: UserWarning: Trying to convert audio automatically from float64 to 16-bit int format.\n", " warnings.warn(warning.format(data.dtype))\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "This is going to be fun, let's enjoy ourselves Darth Wader - Neutral {'text': \"This is going to be fun, let's enjoy ourselves\", 'voice': 'Darth Wader - Neutral'}\n", " > Text splitted to sentences.\n", "[\"This is going to be fun, let's enjoy ourselves\"]\n", " > Processing time: 7.824646234512329\n", " > Real-time factor: 1.8261372721316347\n", "Keyboard interruption in main thread... closing server.\n" ] }, { "data": { "text/plain": [] }, "execution_count": 127, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#INTERFACE WITH AUDIO TO AUDIO\n", "\n", "#to be able to use the microphone on chrome, you will have to go to chrome://flags/#unsafely-treat-insecure-origin-as-secure and enter http://10.186.115.21:7860/ \n", "#in \"Insecure origins treated as secure\", enable it and relaunch chrome\n", "\n", "\n", "model_answer= ''\n", "general_context= \"This is going to be fun, let's enjoy ourselves\"\n", "# Define the initial state with some initial context.\n", "print(general_context)\n", "initial_state = {'context': general_context}\n", "initial_context= initial_state['context']\n", "# Create the Gradio interface.\n", "iface = gr.Interface(\n", " fn=tts_gradio,\n", " inputs=[\n", " gr.Textbox(value=initial_context, visible=True, label='Enter the text to be converted to speech', placeholder=\"This is going to be fun, let's enjoy ourselves\", lines=5),\n", " gr.Radio(choices=voice_options, label='Choose a voice', value=voice_options[0], show_label=True), # Radio button for voice selection\n", " gr.State() # This will keep track of the context state across interactions.\n", " ],\n", " outputs=[\n", " gr.Audio(label = 'output audio', autoplay=True),\n", " gr.State()\n", " ],\n", " flagging_options=['👎', '👍'],\n", ")\n", "#close all interfaces open to make the port available\n", "gr.close_all()\n", "# Launch the interface.\n", "iface.launch(debug=True, share=False, server_name=\"0.0.0.0\", server_port=7860, ssl_verify=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "base", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.8" } }, "nbformat": 4, "nbformat_minor": 2 }