import gradio as gr import pandas as pd from datasets import load_dataset import threading, time, uuid, sqlite3, shutil, os, random, asyncio, threading from pathlib import Path from huggingface_hub import CommitScheduler, delete_file, hf_hub_download from gradio_client import Client import pyloudnorm as pyln import soundfile as sf from detoxify import Detoxify toxicity = Detoxify('original') with open('harvard_sentences.txt') as f: sents = f.read().strip().splitlines() #################################### # Constants #################################### BLOG_POST_LINK = '' # <<<<< ---- AVAILABLE_MODELS = { 'XTTSv2': 'xtts', 'WhisperSpeech': 'whisperspeech', 'ElevenLabs': 'eleven', 'OpenVoice': 'openvoice', 'Pheme': 'pheme', 'MetaVoice': 'metavoice', 'OpenAI': 'openai', } SPACE_ID = os.getenv('HF_ID') MAX_SAMPLE_TXT_LENGTH = 150 MIN_SAMPLE_TXT_LENGTH = 10 DB_DATASET_ID = os.getenv('DATASET_ID') DB_NAME = "database.db" # If /data available => means local storage is enabled => let's use it! DB_PATH = f"/data/{DB_NAME}" if os.path.isdir("/data") else DB_NAME print(f"Using {DB_PATH}") # AUDIO_DATASET_ID = "ttseval/tts-arena-new" CITATION_TEXT = """@misc{tts-arena, title = {Text to Speech Arena}, author = {mrfakename and Srivastav, Vaibhav and Pouget, Lucain and Fourrier, Clémentine and Lacombe, Yoach and main}, year = 2024, publisher = {Hugging Face}, howpublished = "\\url{https://huggingface.co/spaces/ttseval/TTS-Arena}" }""" #################################### # Functions #################################### def create_db_if_missing(): conn = get_db() cursor = conn.cursor() cursor.execute(''' CREATE TABLE IF NOT EXISTS model ( name TEXT UNIQUE, upvote INTEGER, downvote INTEGER ); ''') cursor.execute(''' CREATE TABLE IF NOT EXISTS vote ( id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT, model TEXT, vote INTEGER, timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); ''') cursor.execute(''' CREATE TABLE IF NOT EXISTS votelog ( id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT, chosen TEXT, rejected TEXT, timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); ''') cursor.execute(''' CREATE TABLE IF NOT EXISTS spokentext ( id INTEGER PRIMARY KEY AUTOINCREMENT, spokentext TEXT, timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); ''') def get_db(): return sqlite3.connect(DB_PATH) #################################### # Space initialization #################################### # Download existing DB if not os.path.isfile(DB_PATH): print("Downloading DB...") try: cache_path = hf_hub_download(repo_id=DB_DATASET_ID, repo_type='dataset', filename=DB_NAME) shutil.copyfile(cache_path, DB_PATH) print("Downloaded DB") except Exception as e: print("Error while downloading DB:", e) # Create DB table (if doesn't exist) create_db_if_missing() # Sync local DB with remote repo every 5 minute (only if a change is detected) scheduler = CommitScheduler( repo_id=DB_DATASET_ID, repo_type="dataset", folder_path=Path(DB_PATH).parent, every=5, allow_patterns=DB_NAME, ) # Load audio dataset # audio_dataset = load_dataset(AUDIO_DATASET_ID) #################################### # Router API #################################### router = Client("ttseval/tts-router", hf_token=os.getenv('HF_TOKEN')) #################################### # Gradio app #################################### MUST_BE_LOGGEDIN = "Please login with Hugging Face to participate in the TTS Arena." DESCR = """ # TTS Arena Vote on different speech synthesis models! """.strip() # INSTR = """ # ## Instructions # * Listen to two anonymous models # * Vote on which synthesized audio sounds more natural to you # * If there's a tie, click Skip # **When you're ready to begin, login and begin voting!** The model names will be revealed once you vote. # """.strip() INSTR = """ ## Instructions * Input the text to synthesize (or press the 🎲 for a random input text). * Listen to the two audio clips, one after the other. * Vote on which audio sounds more natural to you. Model names will be revealed after the vote is cast. **When you're ready to begin, enter text.** """.strip() request = '' if SPACE_ID: request = f""" ### Request Model Please fill out [this form](https://huggingface.co/spaces/{SPACE_ID}/discussions/new?title=%5BModel+Request%5D+&description=%23%23%20Model%20Request%0A%0A%2A%2AModel%20website%2Fpaper%20%28if%20applicable%29%2A%2A%3A%0A%2A%2AModel%20available%20on%2A%2A%3A%20%28coqui%7CHF%20pipeline%7Ccustom%20code%29%0A%2A%2AWhy%20do%20you%20want%20this%20model%20added%3F%2A%2A%0A%2A%2AComments%3A%2A%2A) to request a model. """ ABOUT = f""" ## About The TTS Arena is a project created to evaluate leading speech synthesis models. It is inspired by the [Chatbot Arena](https://chat.lmsys.org/) by LMSys. For more information, please check out our [blog post]({BLOG_POST_LINK}) ### Credits Thank you to the following individuals who helped make this project possible: * VB ([Twitter](https://twitter.com/reach_vb) / [Hugging Face](https://huggingface.co/reach-vb)) * Lucain Pouget ([Twitter](https://twitter.com/Wauplin) / [Hugging Face](https://huggingface.co/Wauplin)) * Clémentine Fourrier ([Twitter](https://twitter.com/clefourrier) / [Hugging Face](https://huggingface.co/clefourrier)) * Yoach Lacombe ([Twitter](https://twitter.com/yoachlacombe) / [Hugging Face](https://huggingface.co/ylacombe)) * Main Horse ([Twitter](https://twitter.com/main_horse) / [Hugging Face](https://huggingface.co/main-horse)) * Apolinário Passos ([Twitter](https://twitter.com/multimodalart) / [Hugging Face](https://huggingface.co/multimodalart)) * Sanchit Gandhi ([Twitter](https://twitter.com/sanchitgandhi99) / [Hugging Face](https://huggingface.co/sanchit-gandhi)) * Pedro Cuenca ([Twitter](https://twitter.com/pcuenq) / [Hugging Face](https://huggingface.co/pcuenq)) {request} ### Privacy Statement We may store text you enter and generated audio. We store a unique ID for each session. You agree that we may collect, share, and/or publish any data you input for research and/or commercial purposes. ### License Generated audio clips cannot be redistributed and used for personal, non-commercial use only. """.strip() LDESC = """ ## Leaderboard A list of the models, based on how natural they sound (according to the votes casted)! ### **Important**: To keep a fair impression of results, the leaderboard will be **hidden** by default, until a large number of human ratings have been recorded. Tick the `Reveal Preliminary Results` checkbox below if you wish to see the raw data. """.strip() # def reload_audio_dataset(): # global audio_dataset # audio_dataset = load_dataset(AUDIO_DATASET_ID) # return 'Reload Audio Dataset' def del_db(txt): if not txt.lower() == 'delete db': raise gr.Error('You did not enter "delete db"') # Delete local + remote os.remove(DB_PATH) delete_file(path_in_repo=DB_NAME, repo_id=DB_DATASET_ID, repo_type='dataset') # Recreate create_db_if_missing() return 'Delete DB' theme = gr.themes.Base( font=[gr.themes.GoogleFont('Libre Franklin'), gr.themes.GoogleFont('Public Sans'), 'system-ui', 'sans-serif'], ) model_names = { 'styletts2': 'StyleTTS 2', 'tacotron': 'Tacotron', 'tacotronph': 'Tacotron Phoneme', 'tacotrondca': 'Tacotron DCA', 'speedyspeech': 'Speedy Speech', 'overflow': 'Overflow TTS', 'vits': 'VITS', 'vitsneon': 'VITS Neon', 'neuralhmm': 'Neural HMM', 'glow': 'Glow TTS', 'fastpitch': 'FastPitch', 'jenny': 'Jenny', 'tortoise': 'Tortoise TTS', 'xtts2': 'Coqui XTTSv2', 'xtts': 'Coqui XTTS', 'openvoice': 'MyShell OpenVoice', 'elevenlabs': 'ElevenLabs', 'openai': 'OpenAI', 'hierspeech': 'HierSpeech++', 'pheme': 'PolyAI Pheme', 'speecht5': 'SpeechT5', 'metavoice': 'MetaVoice-1B', } model_licenses = { 'styletts2': 'MIT', 'tacotron': 'BSD-3', 'tacotronph': 'BSD-3', 'tacotrondca': 'BSD-3', 'speedyspeech': 'BSD-3', 'overflow': 'MIT', 'vits': 'MIT', 'openvoice': 'MIT', 'vitsneon': 'BSD-3', 'neuralhmm': 'MIT', 'glow': 'MIT', 'fastpitch': 'Apache 2.0', 'jenny': 'Jenny License', 'tortoise': 'Apache 2.0', 'xtts2': 'CPML (NC)', 'xtts': 'CPML (NC)', 'elevenlabs': 'Proprietary', 'eleven': 'Proprietary', 'openai': 'Proprietary', 'hierspeech': 'MIT', 'pheme': 'CC-BY', 'speecht5': 'MIT', 'metavoice': 'Apache 2.0', 'elevenlabs': 'Proprietary', 'whisperspeech': 'MIT', } model_links = { 'styletts2': 'https://github.com/yl4579/StyleTTS2', 'tacotron': 'https://github.com/NVIDIA/tacotron2', 'speedyspeech': 'https://github.com/janvainer/speedyspeech', 'overflow': 'https://github.com/shivammehta25/OverFlow', 'vits': 'https://github.com/jaywalnut310/vits', 'openvoice': 'https://github.com/myshell-ai/OpenVoice', 'neuralhmm': 'https://github.com/ketranm/neuralHMM', 'glow': 'https://github.com/jaywalnut310/glow-tts', 'fastpitch': 'https://fastpitch.github.io/', 'tortoise': 'https://github.com/neonbjb/tortoise-tts', 'xtts2': 'https://huggingface.co/coqui/XTTS-v2', 'xtts': 'https://huggingface.co/coqui/XTTS-v1', 'elevenlabs': 'https://elevenlabs.io/', 'openai': 'https://help.openai.com/en/articles/8555505-tts-api', 'hierspeech': 'https://github.com/sh-lee-prml/HierSpeechpp', 'pheme': 'https://github.com/PolyAI-LDN/pheme', 'speecht5': 'https://github.com/microsoft/SpeechT5', 'metavoice': 'https://github.com/metavoiceio/metavoice-src', } # def get_random_split(existing_split=None): # choice = random.choice(list(audio_dataset.keys())) # if existing_split and choice == existing_split: # return get_random_split(choice) # else: # return choice # def get_random_splits(): # choice1 = get_random_split() # choice2 = get_random_split(choice1) # return (choice1, choice2) def model_license(name): print(name) for k, v in AVAILABLE_MODELS.items(): if k == name: if v in model_licenses: return model_licenses[v] print('---') return 'Unknown' def get_leaderboard(reveal_prelim: bool): conn = get_db() cursor = conn.cursor() sql = 'SELECT name, upvote, downvote FROM model' # if not reveal_prelim: sql += ' WHERE EXISTS (SELECT 1 FROM model WHERE (upvote + downvote) > 750)' if not reveal_prelim: sql += ' WHERE (upvote + downvote) > 500' cursor.execute(sql) data = cursor.fetchall() df = pd.DataFrame(data, columns=['name', 'upvote', 'downvote']) df['license'] = df['name'].map(model_license) df['name'] = df['name'].replace(model_names) df['votes'] = df['upvote'] + df['downvote'] # df['score'] = round((df['upvote'] / df['votes']) * 100, 2) # Percentage score ## ELO SCORE df['score'] = 1200 for i in range(len(df)): for j in range(len(df)): if i != j: expected_a = 1 / (1 + 10 ** ((df['score'][j] - df['score'][i]) / 400)) expected_b = 1 / (1 + 10 ** ((df['score'][i] - df['score'][j]) / 400)) actual_a = df['upvote'][i] / df['votes'][i] actual_b = df['upvote'][j] / df['votes'][j] df.at[i, 'score'] += 32 * (actual_a - expected_a) df.at[j, 'score'] += 32 * (actual_b - expected_b) df['score'] = round(df['score']) ## ELO SCORE df = df.sort_values(by='score', ascending=False) df['order'] = ['#' + str(i + 1) for i in range(len(df))] # df = df[['name', 'score', 'upvote', 'votes']] df = df[['order', 'name', 'score', 'license', 'votes']] return df def mkuuid(uid): if not uid: uid = uuid.uuid4() return uid def upvote_model(model, uname): conn = get_db() cursor = conn.cursor() cursor.execute('UPDATE model SET upvote = upvote + 1 WHERE name = ?', (model,)) if cursor.rowcount == 0: cursor.execute('INSERT OR REPLACE INTO model (name, upvote, downvote) VALUES (?, 1, 0)', (model,)) cursor.execute('INSERT INTO vote (username, model, vote) VALUES (?, ?, ?)', (uname, model, 1,)) with scheduler.lock: conn.commit() cursor.close() def log_text(text): conn = get_db() cursor = conn.cursor() cursor.execute('INSERT INTO spokentext (spokentext) VALUES (?)', (text,)) with scheduler.lock: conn.commit() cursor.close() def downvote_model(model, uname): conn = get_db() cursor = conn.cursor() cursor.execute('UPDATE model SET downvote = downvote + 1 WHERE name = ?', (model,)) if cursor.rowcount == 0: cursor.execute('INSERT OR REPLACE INTO model (name, upvote, downvote) VALUES (?, 0, 1)', (model,)) cursor.execute('INSERT INTO vote (username, model, vote) VALUES (?, ?, ?)', (uname, model, -1,)) with scheduler.lock: conn.commit() cursor.close() def a_is_better(model1, model2, userid): userid = mkuuid(userid) if model1 and model2: conn = get_db() cursor = conn.cursor() cursor.execute('INSERT INTO votelog (username, chosen, rejected) VALUES (?, ?, ?)', (str(userid), model1, model2,)) with scheduler.lock: conn.commit() cursor.close() upvote_model(model1, str(userid)) downvote_model(model2, str(userid)) return reload(model1, model2, userid, chose_a=True) def b_is_better(model1, model2, userid): userid = mkuuid(userid) if model1 and model2: conn = get_db() cursor = conn.cursor() cursor.execute('INSERT INTO votelog (username, chosen, rejected) VALUES (?, ?, ?)', (str(userid), model2, model1,)) with scheduler.lock: conn.commit() cursor.close() upvote_model(model2, str(userid)) downvote_model(model1, str(userid)) return reload(model1, model2, userid, chose_b=True) def both_bad(model1, model2, userid): userid = mkuuid(userid) if model1 and model2: downvote_model(model1, str(userid)) downvote_model(model2, str(userid)) return reload(model1, model2, userid) def both_good(model1, model2, userid): userid = mkuuid(userid) if model1 and model2: upvote_model(model1, str(userid)) upvote_model(model2, str(userid)) return reload(model1, model2, userid) def reload(chosenmodel1=None, chosenmodel2=None, userid=None, chose_a=False, chose_b=False): # Select random splits # row = random.choice(list(audio_dataset['train'])) # options = list(random.choice(list(audio_dataset['train'])).keys()) # split1, split2 = random.sample(options, 2) # choice1, choice2 = (row[split1], row[split2]) # if chosenmodel1 in model_names: # chosenmodel1 = model_names[chosenmodel1] # if chosenmodel2 in model_names: # chosenmodel2 = model_names[chosenmodel2] # out = [ # (choice1['sampling_rate'], choice1['array']), # (choice2['sampling_rate'], choice2['array']), # split1, # split2 # ] # if userid: out.append(userid) # if chosenmodel1: out.append(f'This model was {chosenmodel1}') # if chosenmodel2: out.append(f'This model was {chosenmodel2}') # return out # return (f'This model was {chosenmodel1}', f'This model was {chosenmodel2}', gr.update(visible=False), gr.update(visible=False)) # return (gr.update(variant='secondary', value=chosenmodel1, interactive=False), gr.update(variant='secondary', value=chosenmodel2, interactive=False)) out = [ gr.update(interactive=False, visible=False), gr.update(interactive=False, visible=False) ] if chose_a == True: out.append(gr.update(value=f'Your vote: {chosenmodel1}', interactive=False, visible=True)) out.append(gr.update(value=f'{chosenmodel2}', interactive=False, visible=True)) else: out.append(gr.update(value=f'{chosenmodel1}', interactive=False, visible=True)) out.append(gr.update(value=f'Your vote: {chosenmodel2}', interactive=False, visible=True)) out.append(gr.update(visible=True)) return out with gr.Blocks() as leaderboard: gr.Markdown(LDESC) # df = gr.Dataframe(interactive=False, value=get_leaderboard()) df = gr.Dataframe(interactive=False, min_width=0, wrap=True, column_widths=[30, 200, 50, 75, 50]) with gr.Row(): reveal_prelim = gr.Checkbox(label="Reveal Preliminary Results", info="Show all models, including models with very few human ratings.", scale=0) reloadbtn = gr.Button("Refresh") reveal_prelim.input(get_leaderboard, inputs=[reveal_prelim], outputs=[df]) leaderboard.load(get_leaderboard, inputs=[reveal_prelim], outputs=[df]) reloadbtn.click(get_leaderboard, inputs=[reveal_prelim], outputs=[df]) gr.Markdown("DISCLAIMER: The licenses listed may not be accurate or up to date, you are responsible for checking the licenses before using the models. Also note that some models may have additional usage restrictions.") # with gr.Blocks() as vote: # useridstate = gr.State() # gr.Markdown(INSTR) # # gr.LoginButton() # with gr.Row(): # gr.HTML('