Spaces:
Running
Running
Commit
•
20dc216
1
Parent(s):
1d45e0d
Vote!
Browse files
app.py
CHANGED
@@ -2,7 +2,8 @@ DESCR = """
|
|
2 |
# TTS Arena
|
3 |
|
4 |
Vote on different speech synthesis models!
|
5 |
-
|
|
|
6 |
## Instructions
|
7 |
|
8 |
* Listen to two anonymous models
|
@@ -13,10 +14,18 @@ Vote on different speech synthesis models!
|
|
13 |
|
14 |
**When you're ready to begin, click the Start button below!** The model names will be revealed once you vote.
|
15 |
""".strip()
|
|
|
|
|
|
|
|
|
|
|
16 |
import gradio as gr
|
17 |
import random
|
18 |
import os
|
|
|
|
|
19 |
from datasets import load_dataset
|
|
|
20 |
dataset = load_dataset("ttseval/tts-arena", token=os.getenv('HF_TOKEN'))
|
21 |
theme = gr.themes.Base(
|
22 |
font=[gr.themes.GoogleFont('Libre Franklin'), gr.themes.GoogleFont('Public Sans'), 'system-ui', 'sans-serif'],
|
@@ -24,6 +33,8 @@ theme = gr.themes.Base(
|
|
24 |
model_names = {
|
25 |
'styletts2': 'StyleTTS 2',
|
26 |
'tacotron': 'Tacotron',
|
|
|
|
|
27 |
'speedyspeech': 'Speedy Speech',
|
28 |
'overflow': 'Overflow TTS',
|
29 |
'vits': 'VITS',
|
@@ -31,6 +42,12 @@ model_names = {
|
|
31 |
'neuralhmm': 'Neural HMM',
|
32 |
'glow': 'Glow TTS',
|
33 |
'fastpitch': 'FastPitch',
|
|
|
|
|
|
|
|
|
|
|
|
|
34 |
}
|
35 |
def get_random_split(existing_split=None):
|
36 |
choice = random.choice(list(dataset.keys()))
|
@@ -38,27 +55,93 @@ def get_random_split(existing_split=None):
|
|
38 |
return get_random_split(choice)
|
39 |
else:
|
40 |
return choice
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
41 |
def get_random_splits():
|
42 |
choice1 = get_random_split()
|
43 |
choice2 = get_random_split(choice1)
|
44 |
return (choice1, choice2)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
45 |
def a_is_better(model1, model2):
|
46 |
-
|
47 |
-
|
48 |
return reload(model1, model2)
|
49 |
def b_is_better(model1, model2):
|
50 |
-
|
51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
52 |
return reload(model1, model2)
|
53 |
def reload(chosenmodel1=None, chosenmodel2=None):
|
54 |
# Select random splits
|
55 |
split1, split2 = get_random_splits()
|
56 |
d1, d2 = (dataset[split1], dataset[split2])
|
57 |
choice1, choice2 = (d1.shuffle()[0]['audio'], d2.shuffle()[0]['audio'])
|
58 |
-
if
|
59 |
-
|
60 |
-
if
|
61 |
-
|
62 |
out = [
|
63 |
(choice1['sampling_rate'], choice1['array']),
|
64 |
(choice2['sampling_rate'], choice2['array']),
|
@@ -68,9 +151,12 @@ def reload(chosenmodel1=None, chosenmodel2=None):
|
|
68 |
if chosenmodel1: out.append(f'This model was {chosenmodel1}')
|
69 |
if chosenmodel2: out.append(f'This model was {chosenmodel2}')
|
70 |
return out
|
71 |
-
with gr.Blocks(
|
72 |
-
|
73 |
-
gr.
|
|
|
|
|
|
|
74 |
with gr.Row():
|
75 |
gr.HTML('<div align="left"><h3>Model A</h3></div>')
|
76 |
gr.HTML('<div align="right"><h3>Model B</h3></div>')
|
@@ -83,13 +169,23 @@ with gr.Blocks(theme=theme, css="footer {visibility: hidden}") as demo:
|
|
83 |
with gr.Row():
|
84 |
aud1 = gr.Audio(interactive=False, show_label=False, show_download_button=False, show_share_button=False, waveform_options={'waveform_progress_color': '#3C82F6'})
|
85 |
aud2 = gr.Audio(interactive=False, show_label=False, show_download_button=False, show_share_button=False, waveform_options={'waveform_progress_color': '#3C82F6'})
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
|
|
|
|
|
|
90 |
outputs = [aud1, aud2, model1, model2, prevmodel1, prevmodel2]
|
91 |
abetter.click(a_is_better, outputs=outputs, inputs=[model1, model2])
|
92 |
bbetter.click(b_is_better, outputs=outputs, inputs=[model1, model2])
|
93 |
skipbtn.click(b_is_better, outputs=outputs, inputs=[model1, model2])
|
94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
95 |
demo.queue(api_open=False).launch(show_api=False)
|
|
|
2 |
# TTS Arena
|
3 |
|
4 |
Vote on different speech synthesis models!
|
5 |
+
""".strip()
|
6 |
+
INSTR = """
|
7 |
## Instructions
|
8 |
|
9 |
* Listen to two anonymous models
|
|
|
14 |
|
15 |
**When you're ready to begin, click the Start button below!** The model names will be revealed once you vote.
|
16 |
""".strip()
|
17 |
+
LDESC = """
|
18 |
+
## Leaderboard
|
19 |
+
|
20 |
+
A list of the models, based on how highly they are ranked!
|
21 |
+
""".strip()
|
22 |
import gradio as gr
|
23 |
import random
|
24 |
import os
|
25 |
+
import pandas as pd
|
26 |
+
import sqlite3
|
27 |
from datasets import load_dataset
|
28 |
+
|
29 |
dataset = load_dataset("ttseval/tts-arena", token=os.getenv('HF_TOKEN'))
|
30 |
theme = gr.themes.Base(
|
31 |
font=[gr.themes.GoogleFont('Libre Franklin'), gr.themes.GoogleFont('Public Sans'), 'system-ui', 'sans-serif'],
|
|
|
33 |
model_names = {
|
34 |
'styletts2': 'StyleTTS 2',
|
35 |
'tacotron': 'Tacotron',
|
36 |
+
'tacotronph': 'Tacotron Phoneme',
|
37 |
+
'tacotrondca': 'Tacotron DCA',
|
38 |
'speedyspeech': 'Speedy Speech',
|
39 |
'overflow': 'Overflow TTS',
|
40 |
'vits': 'VITS',
|
|
|
42 |
'neuralhmm': 'Neural HMM',
|
43 |
'glow': 'Glow TTS',
|
44 |
'fastpitch': 'FastPitch',
|
45 |
+
'jenny': 'Jenny',
|
46 |
+
'tortoise': 'Tortoise TTS',
|
47 |
+
'xtts2': 'XTTSv2',
|
48 |
+
'xtts': 'XTTS',
|
49 |
+
'elevenlabs': 'ElevenLabs',
|
50 |
+
'speecht5': 'SpeechT5',
|
51 |
}
|
52 |
def get_random_split(existing_split=None):
|
53 |
choice = random.choice(list(dataset.keys()))
|
|
|
55 |
return get_random_split(choice)
|
56 |
else:
|
57 |
return choice
|
58 |
+
def get_db():
|
59 |
+
return sqlite3.connect('database.db')
|
60 |
+
def create_db():
|
61 |
+
conn = get_db()
|
62 |
+
cursor = conn.cursor()
|
63 |
+
cursor.execute('''
|
64 |
+
CREATE TABLE IF NOT EXISTS model (
|
65 |
+
name TEXT UNIQUE,
|
66 |
+
upvote INTEGER,
|
67 |
+
downvote INTEGER
|
68 |
+
);
|
69 |
+
''')
|
70 |
+
create_db()
|
71 |
+
def get_data():
|
72 |
+
conn = get_db()
|
73 |
+
cursor = conn.cursor()
|
74 |
+
cursor.execute('SELECT name, upvote, downvote FROM model')
|
75 |
+
data = cursor.fetchall()
|
76 |
+
df = pd.DataFrame(data, columns=['name', 'upvote', 'downvote'])
|
77 |
+
df['name'] = df['name'].replace(model_names)
|
78 |
+
df['votes'] = df['upvote'] + df['downvote']
|
79 |
+
# df['score'] = round((df['upvote'] / df['votes']) * 100, 2) # Percentage score
|
80 |
+
|
81 |
+
## ELO SCORE
|
82 |
+
df['score'] = 1200
|
83 |
+
for i in range(len(df)):
|
84 |
+
for j in range(len(df)):
|
85 |
+
if i != j:
|
86 |
+
expected_a = 1 / (1 + 10 ** ((df['score'][j] - df['score'][i]) / 400))
|
87 |
+
expected_b = 1 / (1 + 10 ** ((df['score'][i] - df['score'][j]) / 400))
|
88 |
+
actual_a = df['upvote'][i] / df['votes'][i]
|
89 |
+
actual_b = df['upvote'][j] / df['votes'][j]
|
90 |
+
df.at[i, 'score'] += 32 * (actual_a - expected_a)
|
91 |
+
df.at[j, 'score'] += 32 * (actual_b - expected_b)
|
92 |
+
df['score'] = round(df['score'])
|
93 |
+
## ELO SCORE
|
94 |
+
|
95 |
+
df = df.sort_values(by='score', ascending=False)
|
96 |
+
# df = df[['name', 'score', 'upvote', 'votes']]
|
97 |
+
df = df[['name', 'score', 'votes']]
|
98 |
+
return df
|
99 |
+
|
100 |
def get_random_splits():
|
101 |
choice1 = get_random_split()
|
102 |
choice2 = get_random_split(choice1)
|
103 |
return (choice1, choice2)
|
104 |
+
def upvote_model(model):
|
105 |
+
conn = get_db()
|
106 |
+
cursor = conn.cursor()
|
107 |
+
cursor.execute('UPDATE model SET upvote = upvote + 1 WHERE name = ?', (model,))
|
108 |
+
if cursor.rowcount == 0:
|
109 |
+
cursor.execute('INSERT OR REPLACE INTO model (name, upvote, downvote) VALUES (?, 1, 0)', (model,))
|
110 |
+
conn.commit()
|
111 |
+
cursor.close()
|
112 |
+
def downvote_model(model):
|
113 |
+
conn = get_db()
|
114 |
+
cursor = conn.cursor()
|
115 |
+
cursor.execute('UPDATE model SET downvote = downvote + 1 WHERE name = ?', (model,))
|
116 |
+
if cursor.rowcount == 0:
|
117 |
+
cursor.execute('INSERT OR REPLACE INTO model (name, upvote, downvote) VALUES (?, 0, 1)', (model,))
|
118 |
+
conn.commit()
|
119 |
+
cursor.close()
|
120 |
def a_is_better(model1, model2):
|
121 |
+
upvote_model(model1)
|
122 |
+
downvote_model(model2)
|
123 |
return reload(model1, model2)
|
124 |
def b_is_better(model1, model2):
|
125 |
+
upvote_model(model2)
|
126 |
+
downvote_model(model1)
|
127 |
+
return reload(model1, model2)
|
128 |
+
def both_bad(model1, model2):
|
129 |
+
downvote_model(model1)
|
130 |
+
downvote_model(model2)
|
131 |
+
return reload(model1, model2)
|
132 |
+
def both_good(model1, model2):
|
133 |
+
upvote_model(model1)
|
134 |
+
upvote_model(model2)
|
135 |
return reload(model1, model2)
|
136 |
def reload(chosenmodel1=None, chosenmodel2=None):
|
137 |
# Select random splits
|
138 |
split1, split2 = get_random_splits()
|
139 |
d1, d2 = (dataset[split1], dataset[split2])
|
140 |
choice1, choice2 = (d1.shuffle()[0]['audio'], d2.shuffle()[0]['audio'])
|
141 |
+
if chosenmodel1 in model_names:
|
142 |
+
chosenmodel1 = model_names[chosenmodel1]
|
143 |
+
if chosenmodel2 in model_names:
|
144 |
+
chosenmodel2 = model_names[chosenmodel2]
|
145 |
out = [
|
146 |
(choice1['sampling_rate'], choice1['array']),
|
147 |
(choice2['sampling_rate'], choice2['array']),
|
|
|
151 |
if chosenmodel1: out.append(f'This model was {chosenmodel1}')
|
152 |
if chosenmodel2: out.append(f'This model was {chosenmodel2}')
|
153 |
return out
|
154 |
+
with gr.Blocks() as leaderboard:
|
155 |
+
gr.Markdown(LDESC)
|
156 |
+
df = gr.Dataframe(interactive=False, value=get_data())
|
157 |
+
leaderboard.load(get_data, outputs=[df])
|
158 |
+
with gr.Blocks() as vote:
|
159 |
+
gr.Markdown(INSTR)
|
160 |
with gr.Row():
|
161 |
gr.HTML('<div align="left"><h3>Model A</h3></div>')
|
162 |
gr.HTML('<div align="right"><h3>Model B</h3></div>')
|
|
|
169 |
with gr.Row():
|
170 |
aud1 = gr.Audio(interactive=False, show_label=False, show_download_button=False, show_share_button=False, waveform_options={'waveform_progress_color': '#3C82F6'})
|
171 |
aud2 = gr.Audio(interactive=False, show_label=False, show_download_button=False, show_share_button=False, waveform_options={'waveform_progress_color': '#3C82F6'})
|
172 |
+
with gr.Row():
|
173 |
+
abetter = gr.Button("A is Better", variant='primary')
|
174 |
+
bbetter = gr.Button("B is Better", variant='primary')
|
175 |
+
with gr.Row():
|
176 |
+
bothbad = gr.Button("Both are Bad", scale=2)
|
177 |
+
skipbtn = gr.Button("Skip", scale=1)
|
178 |
+
bothgood = gr.Button("Both are Good", scale=2)
|
179 |
outputs = [aud1, aud2, model1, model2, prevmodel1, prevmodel2]
|
180 |
abetter.click(a_is_better, outputs=outputs, inputs=[model1, model2])
|
181 |
bbetter.click(b_is_better, outputs=outputs, inputs=[model1, model2])
|
182 |
skipbtn.click(b_is_better, outputs=outputs, inputs=[model1, model2])
|
183 |
+
|
184 |
+
bothbad.click(both_bad, outputs=outputs, inputs=[model1, model2])
|
185 |
+
bothgood.click(both_good, outputs=outputs, inputs=[model1, model2])
|
186 |
+
|
187 |
+
vote.load(reload, outputs=[aud1, aud2, model1, model2])
|
188 |
+
with gr.Blocks(theme=theme, css="footer {visibility: hidden}") as demo:
|
189 |
+
gr.Markdown(DESCR)
|
190 |
+
gr.TabbedInterface([vote, leaderboard], ['Vote', 'Leaderboard'])
|
191 |
demo.queue(api_open=False).launch(show_api=False)
|