Duplicate from kevinwang676/Bark-UI-with-Voice-Cloning

.gitattributes

@@ -0,0 +1,40 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+SE_checkpoint.pth.tar filter=lfs diff=lfs merge=lfs -text
+best_model.pth.tar filter=lfs diff=lfs merge=lfs -text
+nana_longest_vocal.wav filter=lfs diff=lfs merge=lfs -text
+test.wav filter=lfs diff=lfs merge=lfs -text
+reference.wav filter=lfs diff=lfs merge=lfs -text
+ref.wav filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: Bark with Voice Cloning
+emoji: 📊
+colorFrom: purple
+colorTo: purple
+sdk: gradio
+sdk_version: 3.27.0
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: kevinwang676/Bark-UI-with-Voice-Cloning
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

SE_checkpoint.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f96efb20cbeeefd81fd8336d7f0155bf8902f82f9474e58ccb19d9e12345172
+size 44610930

app.py

@@ -0,0 +1,555 @@
+import os
+import sys
+
+os.system("git clone https://github.com/C0untFloyd/bark-gui.git")
+sys.path.append("./bark-gui/")
+
+from cProfile import label
+from distutils.command.check import check
+from doctest import Example
+import gradio as gr
+import numpy as np
+import logging
+import torch
+import pytorch_seed
+import time
+
+from xml.sax import saxutils
+from bark.api import generate_with_settings
+from bark.api import save_as_prompt
+from settings import Settings
+#import nltk
+
+from bark import SAMPLE_RATE
+from bark.clonevoice import clone_voice
+from bark.generation import SAMPLE_RATE, preload_models
+from scipy.io.wavfile import write as write_wav
+from parseinput import split_and_recombine_text, build_ssml, is_ssml, create_clips_from_ssml
+from datetime import datetime
+from tqdm.auto import tqdm
+from id3tagging import add_id3_tag
+
+import shutil
+
+import string
+import argparse
+import json
+
+from TTS.tts.utils.synthesis import synthesis
+from TTS.tts.utils.text.symbols import make_symbols, phonemes, symbols
+try:
+  from TTS.utils.audio import AudioProcessor
+except:
+  from TTS.utils.audio import AudioProcessor
+
+
+from TTS.tts.models import setup_model
+from TTS.config import load_config
+from TTS.tts.models.vits import *
+
+from TTS.tts.utils.speakers import SpeakerManager
+from pydub import AudioSegment
+
+# from google.colab import files
+import librosa
+
+from scipy.io.wavfile import write, read
+
+import subprocess
+
+
+OUTPUTFOLDER = "Outputs"
+
+
+def generate_text_to_speech(text, selected_speaker, text_temp, waveform_temp, eos_prob, quick_generation, complete_settings, seed, progress=gr.Progress(track_tqdm=True)):
+    if text == None or len(text) < 1:
+        raise gr.Error('No text entered!')
+
+    # Chunk the text into smaller pieces then combine the generated audio
+
+    # generation settings
+    if selected_speaker == 'None':
+        selected_speaker = None
+    if seed != None and seed > 2**32 - 1:
+        logger.warning(f"Seed {seed} > 2**32 - 1 (max), setting to random")
+        seed = None
+    if seed == None or seed <= 0:
+        seed = np.random.default_rng().integers(1, 2**32 - 1)
+    assert(0 < seed and seed < 2**32)
+
+    voice_name = selected_speaker
+    use_last_generation_as_history = "Use last generation as history" in complete_settings
+    save_last_generation = "Save generation as Voice" in complete_settings
+    progress(0, desc="Generating")
+
+    silenceshort = np.zeros(int((float(settings.silence_sentence) / 1000.0) * SAMPLE_RATE), dtype=np.float32)  # quarter second of silence
+    silencelong = np.zeros(int((float(settings.silence_speakers) / 1000.0) * SAMPLE_RATE), dtype=np.float32)  # half a second of silence
+    full_generation = None
+
+    all_parts = []
+    complete_text = ""
+    text = text.lstrip()
+    if is_ssml(text):
+        list_speak = create_clips_from_ssml(text)
+        prev_speaker = None
+        for i, clip in tqdm(enumerate(list_speak), total=len(list_speak)):
+            selected_speaker = clip[0]
+            # Add pause break between speakers
+            if i > 0 and selected_speaker != prev_speaker:
+                all_parts += [silencelong.copy()]
+            prev_speaker = selected_speaker
+            text = clip[1]
+            text = saxutils.unescape(text)
+            if selected_speaker == "None":
+                selected_speaker = None
+
+            print(f"\nGenerating Text ({i+1}/{len(list_speak)}) -> {selected_speaker} (Seed {seed}):`{text}`")
+            complete_text += text
+            with pytorch_seed.SavedRNG(seed):
+                audio_array = generate_with_settings(text_prompt=text, voice_name=selected_speaker, semantic_temp=text_temp, coarse_temp=waveform_temp, eos_p=eos_prob)
+                seed = torch.random.initial_seed()
+            if len(list_speak) > 1:
+                filename = create_filename(OUTPUTFOLDER, seed, "audioclip",".wav")
+                save_wav(audio_array, filename)
+                add_id3_tag(filename, text, selected_speaker, seed)
+
+            all_parts += [audio_array]
+    else:
+        texts = split_and_recombine_text(text, settings.input_text_desired_length, settings.input_text_max_length)
+        for i, text in tqdm(enumerate(texts), total=len(texts)):
+            print(f"\nGenerating Text ({i+1}/{len(texts)}) -> {selected_speaker} (Seed {seed}):`{text}`")
+            complete_text += text
+            if quick_generation == True:
+                with pytorch_seed.SavedRNG(seed):
+                    audio_array = generate_with_settings(text_prompt=text, voice_name=selected_speaker, semantic_temp=text_temp, coarse_temp=waveform_temp, eos_p=eos_prob)
+                    seed = torch.random.initial_seed()
+            else:
+                full_output = use_last_generation_as_history or save_last_generation
+                if full_output:
+                    full_generation, audio_array = generate_with_settings(text_prompt=text, voice_name=voice_name, semantic_temp=text_temp, coarse_temp=waveform_temp, eos_p=eos_prob, output_full=True)
+                else:
+                    audio_array = generate_with_settings(text_prompt=text, voice_name=voice_name, semantic_temp=text_temp, coarse_temp=waveform_temp, eos_p=eos_prob)
+
+            # Noticed this in the HF Demo - convert to 16bit int -32767/32767 - most used audio format  
+            # audio_array = (audio_array * 32767).astype(np.int16)
+
+            if len(texts) > 1:
+                filename = create_filename(OUTPUTFOLDER, seed, "audioclip",".wav")
+                save_wav(audio_array, filename)
+                add_id3_tag(filename, text, selected_speaker, seed)
+
+            if quick_generation == False and (save_last_generation == True or use_last_generation_as_history == True):
+                # save to npz
+                voice_name = create_filename(OUTPUTFOLDER, seed, "audioclip", ".npz")
+                save_as_prompt(voice_name, full_generation)
+                if use_last_generation_as_history:
+                    selected_speaker = voice_name
+
+            all_parts += [audio_array]
+            # Add short pause between sentences
+            if text[-1] in "!?.\n" and i > 1:
+                all_parts += [silenceshort.copy()]
+
+    # save & play audio
+    result = create_filename(OUTPUTFOLDER, seed, "final",".wav")
+    save_wav(np.concatenate(all_parts), result)
+    # write id3 tag with text truncated to 60 chars, as a precaution...
+    add_id3_tag(result, complete_text, selected_speaker, seed)
+    return result
+
+def create_filename(path, seed, name, extension):
+    now = datetime.now()
+    date_str =now.strftime("%m-%d-%Y")
+    outputs_folder = os.path.join(os.getcwd(), path)
+    if not os.path.exists(outputs_folder):
+        os.makedirs(outputs_folder)
+
+    sub_folder = os.path.join(outputs_folder, date_str)
+    if not os.path.exists(sub_folder):
+        os.makedirs(sub_folder)
+
+    time_str = now.strftime("%H-%M-%S")
+    file_name = f"{name}_{time_str}_s{seed}{extension}"
+    return os.path.join(sub_folder, file_name)
+
+
+def save_wav(audio_array, filename):
+    write_wav(filename, SAMPLE_RATE, audio_array)
+
+def save_voice(filename, semantic_prompt, coarse_prompt, fine_prompt):
+    np.savez_compressed(
+        filename,
+        semantic_prompt=semantic_prompt,
+        coarse_prompt=coarse_prompt,
+        fine_prompt=fine_prompt
+    )
+    
+
+def on_quick_gen_changed(checkbox):
+    if checkbox == False:
+        return gr.CheckboxGroup.update(visible=True)
+    return gr.CheckboxGroup.update(visible=False)
+
+def delete_output_files(checkbox_state):
+    if checkbox_state:
+        outputs_folder = os.path.join(os.getcwd(), OUTPUTFOLDER)
+        if os.path.exists(outputs_folder):
+            purgedir(outputs_folder)
+    return False
+
+
+# https://stackoverflow.com/a/54494779
+def purgedir(parent):
+    for root, dirs, files in os.walk(parent):                                      
+        for item in files:
+            # Delete subordinate files                                                 
+            filespec = os.path.join(root, item)
+            os.unlink(filespec)
+        for item in dirs:
+            # Recursively perform this operation for subordinate directories   
+            purgedir(os.path.join(root, item))
+
+def convert_text_to_ssml(text, selected_speaker):
+    return build_ssml(text, selected_speaker)
+
+
+def apply_settings(themes, input_server_name, input_server_port, input_server_public, input_desired_len, input_max_len, input_silence_break, input_silence_speaker):
+    settings.selected_theme = themes
+    settings.server_name = input_server_name
+    settings.server_port = input_server_port
+    settings.server_share = input_server_public
+    settings.input_text_desired_length = input_desired_len
+    settings.input_text_max_length = input_max_len
+    settings.silence_sentence = input_silence_break
+    settings.silence_speaker = input_silence_speaker
+    settings.save()
+
+def restart():
+    global restart_server
+    restart_server = True
+
+
+def create_version_html():
+    python_version = ".".join([str(x) for x in sys.version_info[0:3]])
+    versions_html = f"""
+python: <span title="{sys.version}">{python_version}</span>
+ • 
+torch: {getattr(torch, '__long_version__',torch.__version__)}
+ • 
+gradio: {gr.__version__}
+"""
+    return versions_html
+
+    
+
+logger = logging.getLogger(__name__)
+APPTITLE = "Bark UI Enhanced v0.4.6"
+
+
+autolaunch = False
+
+if len(sys.argv) > 1:
+    autolaunch = "-autolaunch" in sys.argv
+
+
+if torch.cuda.is_available() == False:
+    os.environ['BARK_FORCE_CPU'] = 'True'
+    logger.warning("No CUDA detected, fallback to CPU!")
+
+print(f'smallmodels={os.environ.get("SUNO_USE_SMALL_MODELS", False)}')
+print(f'enablemps={os.environ.get("SUNO_ENABLE_MPS", False)}')
+print(f'offloadcpu={os.environ.get("SUNO_OFFLOAD_CPU", False)}')
+print(f'forcecpu={os.environ.get("BARK_FORCE_CPU", False)}')
+print(f'autolaunch={autolaunch}\n\n')
+
+#print("Updating nltk\n")
+#nltk.download('punkt')
+
+print("Preloading Models\n")
+preload_models()
+
+settings = Settings('config.yaml')
+
+# Collect all existing speakers/voices in dir
+speakers_list = []
+
+for root, dirs, files in os.walk("./bark/assets/prompts"):
+	for file in files:
+		if(file.endswith(".npz")):
+			pathpart = root.replace("./bark/assets/prompts", "")
+			name = os.path.join(pathpart, file[:-4])
+			if name.startswith("/") or name.startswith("\\"):
+				name = name[1:]
+			speakers_list.append(name)
+
+speakers_list = sorted(speakers_list, key=lambda x: x.lower())
+speakers_list.insert(0, 'None')
+
+available_themes = ["Default", "gradio/glass", "gradio/monochrome", "gradio/seafoam", "gradio/soft", "gstaff/xkcd", "freddyaboulton/dracula_revamped", "ysharma/steampunk"]
+
+seed = -1
+server_name = settings.server_name
+if len(server_name) < 1:
+    server_name = None
+server_port = settings.server_port
+if server_port <= 0:
+    server_port = None
+global run_server
+global restart_server
+
+run_server = True
+
+
+
+
+'''
+from google.colab import drive
+drive.mount('/content/drive')
+src_path = os.path.join(os.path.join(os.path.join(os.path.join(os.getcwd(), 'drive'), 'MyDrive'), 'Colab Notebooks'), 'best_model_latest.pth.tar')
+dst_path = os.path.join(os.getcwd(), 'best_model.pth.tar')
+shutil.copy(src_path, dst_path)
+'''
+
+TTS_PATH = "TTS/"
+
+# add libraries into environment
+sys.path.append(TTS_PATH) # set this if TTS is not installed globally
+
+# Paths definition
+
+OUT_PATH = 'out/'
+
+# create output path
+os.makedirs(OUT_PATH, exist_ok=True)
+
+# model vars 
+MODEL_PATH = 'best_model.pth.tar'
+CONFIG_PATH = 'config.json'
+TTS_LANGUAGES = "language_ids.json"
+TTS_SPEAKERS = "speakers.json"
+USE_CUDA = torch.cuda.is_available()
+
+# load the config
+C = load_config(CONFIG_PATH)
+
+# load the audio processor
+ap = AudioProcessor(**C.audio)
+
+speaker_embedding = None
+
+C.model_args['d_vector_file'] = TTS_SPEAKERS
+C.model_args['use_speaker_encoder_as_loss'] = False
+
+model = setup_model(C)
+model.language_manager.set_language_ids_from_file(TTS_LANGUAGES)
+# print(model.language_manager.num_languages, model.embedded_language_dim)
+# print(model.emb_l)
+cp = torch.load(MODEL_PATH, map_location=torch.device('cpu'))
+# remove speaker encoder
+model_weights = cp['model'].copy()
+for key in list(model_weights.keys()):
+  if "speaker_encoder" in key:
+    del model_weights[key]
+
+model.load_state_dict(model_weights)
+
+model.eval()
+
+if USE_CUDA:
+    model = model.cuda()
+
+# synthesize voice
+use_griffin_lim = False
+
+# Paths definition
+
+CONFIG_SE_PATH = "config_se.json"
+CHECKPOINT_SE_PATH = "SE_checkpoint.pth.tar"
+
+# Load the Speaker encoder
+
+SE_speaker_manager = SpeakerManager(encoder_model_path=CHECKPOINT_SE_PATH, encoder_config_path=CONFIG_SE_PATH, use_cuda=USE_CUDA)
+
+# Define helper function
+
+def compute_spec(ref_file):
+  y, sr = librosa.load(ref_file, sr=ap.sample_rate)
+  spec = ap.spectrogram(y)
+  spec = torch.FloatTensor(spec).unsqueeze(0)
+  return spec
+
+
+def voice_conversion(ta, ra, da):
+
+  target_audio = 'target.wav'
+  reference_audio = 'reference.wav'
+  driving_audio = 'driving.wav'
+
+  write(target_audio, ta[0], ta[1])
+  write(reference_audio, ra[0], ra[1])
+  write(driving_audio, da[0], da[1])          
+
+  # !ffmpeg-normalize $target_audio -nt rms -t=-27 -o $target_audio -ar 16000 -f
+  # !ffmpeg-normalize $reference_audio -nt rms -t=-27 -o $reference_audio -ar 16000 -f
+  # !ffmpeg-normalize $driving_audio -nt rms -t=-27 -o $driving_audio -ar 16000 -f
+
+  files = [target_audio, reference_audio, driving_audio]
+
+  for file in files:
+      subprocess.run(["ffmpeg-normalize", file, "-nt", "rms", "-t=-27", "-o", file, "-ar", "16000", "-f"])
+
+  # ta_ = read(target_audio)
+
+  target_emb = SE_speaker_manager.compute_d_vector_from_clip([target_audio])
+  target_emb = torch.FloatTensor(target_emb).unsqueeze(0)
+
+  driving_emb = SE_speaker_manager.compute_d_vector_from_clip([reference_audio])
+  driving_emb = torch.FloatTensor(driving_emb).unsqueeze(0)
+
+  # Convert the voice
+
+  driving_spec = compute_spec(driving_audio)
+  y_lengths = torch.tensor([driving_spec.size(-1)])
+  if USE_CUDA:
+      ref_wav_voc, _, _ = model.voice_conversion(driving_spec.cuda(), y_lengths.cuda(), driving_emb.cuda(), target_emb.cuda())
+      ref_wav_voc = ref_wav_voc.squeeze().cpu().detach().numpy()
+  else:
+      ref_wav_voc, _, _ = model.voice_conversion(driving_spec, y_lengths, driving_emb, target_emb)
+      ref_wav_voc = ref_wav_voc.squeeze().detach().numpy()
+
+  # print("Reference Audio after decoder:")
+  # IPython.display.display(Audio(ref_wav_voc, rate=ap.sample_rate))
+
+  return (ap.sample_rate, ref_wav_voc)
+
+
+while run_server:
+    print(f'Launching {APPTITLE} Server')
+
+    # Create Gradio Blocks
+
+    with gr.Blocks(title=f"{APPTITLE}", mode=f"{APPTITLE}", theme=settings.selected_theme) as barkgui:
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown(f"### [{APPTITLE}](https://github.com/C0untFloyd/bark-gui)")
+            with gr.Column():
+                gr.HTML(create_version_html(), elem_id="versions")
+
+        with gr.Tab("TTS"):
+            with gr.Row():
+                with gr.Column():
+                    placeholder = "Enter text here."
+                    input_text = gr.Textbox(label="Input Text", lines=4, placeholder=placeholder)
+                with gr.Column():
+                    seedcomponent = gr.Number(label="Seed (default -1 = Random)", precision=0, value=-1)
+                    convert_to_ssml_button = gr.Button("Convert Text to SSML")
+            with gr.Row():
+                with gr.Column():
+                    examples = [
+                        "Special meanings: [laughter] [laughs] [sighs] [music] [gasps] [clears throat] MAN: WOMAN:",
+                       "♪ Never gonna make you cry, never gonna say goodbye, never gonna tell a lie and hurt you ♪",
+                       "And now — a picture of a larch [laughter]",
+                       """
+                            WOMAN: I would like an oatmilk latte please.
+                            MAN: Wow, that's expensive!
+                       """,
+                       """<?xml version="1.0"?>
+    <speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis"
+             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+             xsi:schemaLocation="http://www.w3.org/2001/10/synthesis
+                       http://www.w3.org/TR/speech-synthesis/synthesis.xsd"
+             xml:lang="en-US">
+    <voice name="en_speaker_9">Look at that drunk guy!</voice>
+    <voice name="en_speaker_3">Who is he?</voice>
+    <voice name="en_speaker_9">WOMAN: [clears throat] 10 years ago, he proposed me and I rejected him.</voice>
+    <voice name="en_speaker_3">Oh my God [laughs] he is still celebrating</voice>
+    </speak>"""
+                       ]
+                    examples = gr.Examples(examples=examples, inputs=input_text)
+
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("[Voice Prompt Library](https://suno-ai.notion.site/8b8e8749ed514b0cbf3f699013548683?v=bc67cff786b04b50b3ceb756fd05f68c)")
+                    speaker = gr.Dropdown(speakers_list, value=speakers_list[0], label="Voice")
+                with gr.Column():
+                    text_temp = gr.Slider(0.1, 1.0, value=0.6, label="Generation Temperature", info="1.0 more diverse, 0.1 more conservative")
+                    waveform_temp = gr.Slider(0.1, 1.0, value=0.7, label="Waveform temperature", info="1.0 more diverse, 0.1 more conservative")
+
+            with gr.Row():
+                with gr.Column():
+                    quick_gen_checkbox = gr.Checkbox(label="Quick Generation", value=True)
+                    settings_checkboxes = ["Use last generation as history", "Save generation as Voice"]
+                    complete_settings = gr.CheckboxGroup(choices=settings_checkboxes, value=settings_checkboxes, label="Detailed Generation Settings", type="value", interactive=True, visible=False)
+                with gr.Column():
+                    eos_prob = gr.Slider(0.0, 0.5, value=0.05, label="End of sentence probability")
+
+            with gr.Row():
+                with gr.Column():
+                    tts_create_button = gr.Button("Generate")
+                with gr.Column():
+                    hidden_checkbox = gr.Checkbox(visible=False)
+                    button_stop_generation = gr.Button("Stop generation")
+            with gr.Row():
+                output_audio = gr.Audio(label="Generated Audio")
+
+            with gr.Row():
+                inp1 = gr.Audio(label='Target Speaker - Reference Clip')
+                inp2 = output_audio
+                inp3 = output_audio
+                btn = gr.Button("Generate")
+                out1 = gr.Audio(label='Target Speaker - Converted Clip')
+            btn.click(voice_conversion, [inp1, inp2, inp3], [out1])
+                
+
+
+        with gr.Tab("Clone Voice"):
+            input_audio_filename = gr.Audio(label="Input audio.wav", source="upload", type="filepath")
+            transcription_text = gr.Textbox(label="Transcription Text", lines=1, placeholder="Enter Text of your Audio Sample here...")
+            initialname = "./bark/assets/prompts/custom/MeMyselfAndI"
+            output_voice = gr.Textbox(label="Filename of trained Voice", lines=1, placeholder=initialname, value=initialname)
+            clone_voice_button = gr.Button("Create Voice")
+            dummy = gr.Text(label="Progress")
+
+        with gr.Tab("Settings"):
+            with gr.Row():
+                themes = gr.Dropdown(available_themes, label="Theme", info="Change needs complete restart", value=settings.selected_theme)
+            with gr.Row():
+                input_server_name = gr.Textbox(label="Server Name", lines=1, info="Leave blank to run locally", value=settings.server_name)
+                input_server_port = gr.Number(label="Server Port", precision=0, info="Leave at 0 to use default", value=settings.server_port)
+                share_checkbox = gr.Checkbox(label="Public Server", value=settings.server_share)
+            with gr.Row():
+                input_desired_len = gr.Slider(100, 150, value=settings.input_text_desired_length, label="Desired Input Text Length", info="Ideal length to split input sentences")
+                input_max_len = gr.Slider(150, 256, value=settings.input_text_max_length, label="Max Input Text Length", info="Maximum Input Text Length")
+            with gr.Row():
+                input_silence_break = gr.Slider(1, 1000, value=settings.silence_sentence, label="Sentence Pause Time (ms)", info="Silence between sentences in milliseconds")
+                input_silence_speakers = gr.Slider(1, 5000, value=settings.silence_speakers, label="Speaker Pause Time (ms)", info="Silence between different speakers in milliseconds")
+
+            with gr.Row():
+                button_apply_settings = gr.Button("Apply Settings")
+                button_apply_restart = gr.Button("Restart Server")
+                button_delete_files = gr.Button("Clear output folder")
+
+        quick_gen_checkbox.change(fn=on_quick_gen_changed, inputs=quick_gen_checkbox, outputs=complete_settings)
+        convert_to_ssml_button.click(convert_text_to_ssml, inputs=[input_text, speaker],outputs=input_text)
+        gen_click = tts_create_button.click(generate_text_to_speech, inputs=[input_text, speaker, text_temp, waveform_temp, eos_prob, quick_gen_checkbox, complete_settings, seedcomponent],outputs=output_audio)
+        button_stop_generation.click(fn=None, inputs=None, outputs=None, cancels=[gen_click])
+        # Javascript hack to display modal confirmation dialog
+        js = "(x) => confirm('Are you sure? This will remove all files from output folder')"
+        button_delete_files.click(None, None, hidden_checkbox, _js=js)
+        hidden_checkbox.change(delete_output_files, [hidden_checkbox], [hidden_checkbox])
+        clone_voice_button.click(clone_voice, inputs=[input_audio_filename, transcription_text, output_voice], outputs=dummy)
+        button_apply_settings.click(apply_settings, inputs=[themes, input_server_name, input_server_port, share_checkbox, input_desired_len, input_max_len, input_silence_break, input_silence_speakers])
+        button_apply_restart.click(restart)
+        restart_server = False
+        try:
+            barkgui.queue().launch(show_error=True)
+        except:
+            restart_server = True
+            run_server = False
+        try:
+            while restart_server == False:
+                time.sleep(1.0)
+        except (KeyboardInterrupt, OSError):
+            print("Keyboard interruption in main thread... closing server.")
+            run_server = False
+        barkgui.close()

bark/__init__.py

@@ -0,0 +1,2 @@
+from .api import generate_audio, text_to_semantic, semantic_to_waveform, save_as_prompt
+from .generation import SAMPLE_RATE, preload_models

bark/api.py

@@ -0,0 +1,158 @@
+from typing import Dict, Optional, Union
+
+import numpy as np
+
+from .generation import codec_decode, generate_coarse, generate_fine, generate_text_semantic
+
+
+def generate_with_settings(text_prompt, semantic_temp=0.6, eos_p=0.2, coarse_temp=0.7, fine_temp=0.5, voice_name=None, output_full=False):
+
+    # generation with more control
+    x_semantic = generate_text_semantic(
+        text_prompt,
+        history_prompt=voice_name,
+        temp=semantic_temp,
+        min_eos_p = eos_p,
+        use_kv_caching=True
+    )
+
+    x_coarse_gen = generate_coarse(
+        x_semantic,
+        history_prompt=voice_name,
+        temp=coarse_temp,
+        use_kv_caching=True
+    )
+    x_fine_gen = generate_fine(
+        x_coarse_gen,
+        history_prompt=voice_name,
+        temp=fine_temp,
+    )
+
+    if output_full:
+        full_generation = {
+            'semantic_prompt': x_semantic,
+            'coarse_prompt': x_coarse_gen,
+            'fine_prompt': x_fine_gen
+        }
+        return full_generation, codec_decode(x_fine_gen)
+    return codec_decode(x_fine_gen)
+
+
+def text_to_semantic(
+    text: str,
+    history_prompt: Optional[Union[Dict, str]] = None,
+    temp: float = 0.7,
+    silent: bool = False,
+):
+    """Generate semantic array from text.
+
+    Args:
+        text: text to be turned into audio
+        history_prompt: history choice for audio cloning
+        temp: generation temperature (1.0 more diverse, 0.0 more conservative)
+        silent: disable progress bar
+
+    Returns:
+        numpy semantic array to be fed into `semantic_to_waveform`
+    """
+    x_semantic = generate_text_semantic(
+        text,
+        history_prompt=history_prompt,
+        temp=temp,
+        silent=silent,
+        use_kv_caching=True
+    )
+    return x_semantic
+
+
+def semantic_to_waveform(
+    semantic_tokens: np.ndarray,
+    history_prompt: Optional[Union[Dict, str]] = None,
+    temp: float = 0.7,
+    silent: bool = False,
+    output_full: bool = False,
+):
+    """Generate audio array from semantic input.
+
+    Args:
+        semantic_tokens: semantic token output from `text_to_semantic`
+        history_prompt: history choice for audio cloning
+        temp: generation temperature (1.0 more diverse, 0.0 more conservative)
+        silent: disable progress bar
+        output_full: return full generation to be used as a history prompt
+
+    Returns:
+        numpy audio array at sample frequency 24khz
+    """
+    coarse_tokens = generate_coarse(
+        semantic_tokens,
+        history_prompt=history_prompt,
+        temp=temp,
+        silent=silent,
+        use_kv_caching=True
+    )
+    fine_tokens = generate_fine(
+        coarse_tokens,
+        history_prompt=history_prompt,
+        temp=0.5,
+    )
+    audio_arr = codec_decode(fine_tokens)
+    if output_full:
+        full_generation = {
+            "semantic_prompt": semantic_tokens,
+            "coarse_prompt": coarse_tokens,
+            "fine_prompt": fine_tokens,
+        }
+        return full_generation, audio_arr
+    return audio_arr
+
+
+def save_as_prompt(filepath, full_generation):
+    assert(filepath.endswith(".npz"))
+    assert(isinstance(full_generation, dict))
+    assert("semantic_prompt" in full_generation)
+    assert("coarse_prompt" in full_generation)
+    assert("fine_prompt" in full_generation)
+    np.savez(filepath, **full_generation)
+
+
+def generate_audio(
+    text: str,
+    history_prompt: Optional[Union[Dict, str]] = None,
+    text_temp: float = 0.7,
+    waveform_temp: float = 0.7,
+    silent: bool = False,
+    output_full: bool = False,
+):
+    """Generate audio array from input text.
+
+    Args:
+        text: text to be turned into audio
+        history_prompt: history choice for audio cloning
+        text_temp: generation temperature (1.0 more diverse, 0.0 more conservative)
+        waveform_temp: generation temperature (1.0 more diverse, 0.0 more conservative)
+        silent: disable progress bar
+        output_full: return full generation to be used as a history prompt
+
+    Returns:
+        numpy audio array at sample frequency 24khz
+    """
+    semantic_tokens = text_to_semantic(
+        text,
+        history_prompt=history_prompt,
+        temp=text_temp,
+        silent=silent,
+    )
+    out = semantic_to_waveform(
+        semantic_tokens,
+        history_prompt=history_prompt,
+        temp=waveform_temp,
+        silent=silent,
+        output_full=output_full,
+    )
+    if output_full:
+        full_generation, audio_arr = out
+        return full_generation, audio_arr
+    else:
+        audio_arr = out
+    return audio_arr

bark/assets/prompts/announcer.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26f2d1a9e3b6fe453cf5fc8191de26cbfae6276c5b0f7c376c6a0f3c35867f83
+size 16794

bark/assets/prompts/en_speaker_0.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:932f40d879ba8659f1ca26319ba64ea3b0647b2050fe24313bf42b0dff1fe241
+size 28100

bark/assets/prompts/en_speaker_1.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e7f18015e1ab9b6302ded1e28a971af5306a72f193bb6c411f1948a083c8578
+size 25220

bark/assets/prompts/en_speaker_2.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d218990680ece5f2d4fc18ea4783b016b3ae353ec413eaee2058f2d57263c9b3
+size 26236

bark/assets/prompts/en_speaker_3.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92c2e2a29145c83738e9b63f082fd1c873d9422468a155463cb27f814aeaea66
+size 34980

bark/assets/prompts/en_speaker_4.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:992f91991a9a5359d72f00b09a11a550e71bb8ebfc0cfd877e39d7d41f98b714
+size 23780

bark/assets/prompts/en_speaker_5.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18831c3f6014e4a2ff60ad5169b1fae06e28ed07f43f8a3616aafb84515091bf
+size 24740

bark/assets/prompts/en_speaker_6.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fab38dc6b6bc9226bcc414f4c5a9524bc1b2441865a586153fb620127a8faa4e
+size 25540

bark/assets/prompts/en_speaker_7.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f4c4eb33f5994be8de5cfd1744ebce13da1618a6da3a7d244514178c61ef7db
+size 22716

bark/assets/prompts/en_speaker_8.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fc9f11b539588f51bbf78150a73e0365c49b2306bd72e5a22b28ef09c4fb15d
+size 23300

bark/assets/prompts/en_speaker_9.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78b3ba32eb9aeb9ed34556856c40633ecc8332d1c3ae3c81e6f5015ac3eefbd5
+size 30180

bark/assets/prompts/zh_speaker_0.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd7ac118a3e944b3f20c89f2446056a00850a630ee16318922acc6572ce80929
+size 20636

bark/assets/prompts/zh_speaker_1.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0eacf5c862dfd3c5ac825f2ebb26f323e64309cb712e7e264cbd31c5bca3f038
+size 19836

bark/assets/prompts/zh_speaker_2.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e324b47f8250e5798c314f395d4e049575e7ca369d0b6074e91c7bba70e9f26d
+size 21060

bark/assets/prompts/zh_speaker_3.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98c476abc7bf634ffb2d71d363284e7bd8c8abd5e33ec5ca21d4aa5b15730d18
+size 31300

bark/assets/prompts/zh_speaker_4.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fa8673a9895ad3302d13ac94193b5ad5da481f1cc276e6181fa895acaae133b
+size 29964

bark/assets/prompts/zh_speaker_5.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:226edfe5fabc72eeb83a13e350599bc8babe5adc2264b3cdb661fd1258dc4044
+size 17436

bark/assets/prompts/zh_speaker_6.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:285d51fbe81cc263636b5b487fbb6633e6f3cf92c53ca9ab8e6b7f55d4b4a31d
+size 16900

bark/assets/prompts/zh_speaker_7.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0967cdb14ffa79895747b0d52df9f15bdad80d6c55b7630894345c9a7ec87c91
+size 21060

bark/assets/prompts/zh_speaker_8.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c028f78530013f29ab8c0c1cf4fe2138106fbe5252951f5f36e0168056779549
+size 19300

bark/assets/prompts/zh_speaker_9.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6265bb827008d7af8a45a8e057fe3e91efb347d56208180a9ed990ad54e4d75e
+size 16156

bark/clonevoice.py

@@ -0,0 +1,41 @@
+from bark.generation import load_codec_model, generate_text_semantic, grab_best_device
+from encodec.utils import convert_audio
+import torchaudio
+import torch
+import os
+import gradio
+
+
+def clone_voice(audio_filepath, text, dest_filename, progress=gradio.Progress(track_tqdm=True)):
+    if len(text) < 1:
+        raise gradio.Error('No transcription text entered!')
+
+    use_gpu = not os.environ.get("BARK_FORCE_CPU", False)
+    progress(0, desc="Loading Codec")
+    model = load_codec_model(use_gpu=use_gpu)
+    progress(0.25, desc="Converting WAV")
+
+    # Load and pre-process the audio waveform
+    device = grab_best_device(use_gpu)
+    wav, sr = torchaudio.load(audio_filepath)
+    wav = convert_audio(wav, sr, model.sample_rate, model.channels)
+    wav = wav.unsqueeze(0).to(device)
+    progress(0.5, desc="Extracting codes")
+
+    # Extract discrete codes from EnCodec
+    with torch.no_grad():
+        encoded_frames = model.encode(wav)
+    codes = torch.cat([encoded[0] for encoded in encoded_frames], dim=-1).squeeze()  # [n_q, T]
+
+    # get seconds of audio
+    seconds = wav.shape[-1] / model.sample_rate
+    # generate semantic tokens
+    semantic_tokens = generate_text_semantic(text, max_gen_duration_s=seconds, top_k=50, top_p=.95, temp=0.7)
+
+    # move codes to cpu
+    codes = codes.cpu().numpy()
+
+    import numpy as np
+    output_path = dest_filename + '.npz'
+    np.savez(output_path, fine_prompt=codes, coarse_prompt=codes[:2, :], semantic_prompt=semantic_tokens)
+    return "Finished"