pyp1 commited on
Commit
ecf2fd0
1 Parent(s): 0d18e61

better handle numbers

Browse files
app.py CHANGED
@@ -200,6 +200,16 @@ def get_output_audio(audio_tensors, codec_audio_sr):
200
  buffer.seek(0)
201
  return buffer.read()
202
 
 
 
 
 
 
 
 
 
 
 
203
  @spaces.GPU(duration=90)
204
  def run(seed, left_margin, right_margin, codec_audio_sr, codec_sr, top_k, top_p, temperature,
205
  stop_repetition, sample_batch_size, kvcache, silence_tokens,
@@ -212,6 +222,8 @@ def run(seed, left_margin, right_margin, codec_audio_sr, codec_sr, top_k, top_p,
212
  raise gr.Error("Can't use smart transcript: whisper transcript not found")
213
 
214
  seed_everything(seed)
 
 
215
  if mode == "Long TTS":
216
  if split_text == "Newline":
217
  sentences = transcript.split('\n')
@@ -370,32 +382,32 @@ If disabled, you should write the target transcript yourself:</br>
370
  - In Edit mode write full prompt</br>
371
  """
372
 
373
- demo_original_transcript = "And again in two thousand and eight when the United States Central Bank, the Federal Reserve, printed over two trillion dollars."
374
 
375
  demo_text = {
376
  "TTS": {
377
  "smart": "I cannot believe that the same model can also do text to speech synthesis too!",
378
- "regular": "And again in two thousand and eight when the united states central bank, I cannot believe that the same model can also do text to speech synthesis too!"
379
  },
380
  "Edit": {
381
- "smart": "Central Bank of the United States, also called",
382
- "regular": "And again in two thousand and eight when the Central Bank of the United States, also called the Federal Reserve, printed over two trillion dollars."
383
  },
384
  "Long TTS": {
385
  "smart": "You can run the model on a big text!\n"
386
  "Just write it line-by-line. Or sentence-by-sentence.\n"
387
  "If some sentences sound odd, just rerun the model on them, no need to generate the whole text again!",
388
- "regular": "And again in two thousand and eight when the united states central bank, You can run the model on a big text!\n"
389
- "And again in two thousand and eight when the united states central bank, Just write it line-by-line. Or sentence-by-sentence.\n"
390
- "And again in two thousand and eight when the united states central bank, If some sentences sound odd, just rerun the model on them, no need to generate the whole text again!"
391
  }
392
  }
393
 
394
  all_demo_texts = {vv for k, v in demo_text.items() for kk, vv in v.items()}
395
 
396
- demo_words = ['0.12 And 0.221', '0.261 again 0.561', '0.622 in 0.682', '0.742 two 0.922', '0.983 thousand 1.464', '1.504 and 1.584', '1.684 eight 1.865', '1.945 when 2.085', '2.125 the 2.206', '2.266 United 2.667', '2.707 States 2.968', '3.008 Central 3.349', '3.389 Bank, 3.649', '3.83 the 3.93', '4.01 Federal 4.451', '4.532 Reserve, 5.113', '5.314 printed 5.674', '5.835 over 6.035', '6.176 two 6.517', '6.637 trillion 7.098', '7.118 dollars. 7.479']
397
 
398
- demo_words_info = [{'word': 'And', 'start': 0.12, 'end': 0.221, 'score': 0.792}, {'word': 'again', 'start': 0.261, 'end': 0.561, 'score': 0.795}, {'word': 'in', 'start': 0.622, 'end': 0.682, 'score': 0.75}, {'word': 'two', 'start': 0.742, 'end': 0.922, 'score': 0.755}, {'word': 'thousand', 'start': 0.983, 'end': 1.464, 'score': 0.82}, {'word': 'and', 'start': 1.504, 'end': 1.584, 'score': 0.715}, {'word': 'eight', 'start': 1.684, 'end': 1.865, 'score': 0.885}, {'word': 'when', 'start': 1.945, 'end': 2.085, 'score': 0.987}, {'word': 'the', 'start': 2.125, 'end': 2.206, 'score': 0.833}, {'word': 'United', 'start': 2.266, 'end': 2.667, 'score': 0.818}, {'word': 'States', 'start': 2.707, 'end': 2.968, 'score': 0.842}, {'word': 'Central', 'start': 3.008, 'end': 3.349, 'score': 0.852}, {'word': 'Bank,', 'start': 3.389, 'end': 3.649, 'score': 0.98}, {'word': 'the', 'start': 3.83, 'end': 3.93, 'score': 0.996}, {'word': 'Federal', 'start': 4.01, 'end': 4.451, 'score': 0.795}, {'word': 'Reserve,', 'start': 4.532, 'end': 5.113, 'score': 0.852}, {'word': 'printed', 'start': 5.314, 'end': 5.674, 'score': 0.785}, {'word': 'over', 'start': 5.835, 'end': 6.035, 'score': 0.84}, {'word': 'two', 'start': 6.176, 'end': 6.517, 'score': 0.757}, {'word': 'trillion', 'start': 6.637, 'end': 7.098, 'score': 0.796}, {'word': 'dollars.', 'start': 7.118, 'end': 7.479, 'score': 0.939}]
399
 
400
 
401
  def update_demo(mode, smart_transcript, edit_word_mode, transcript, edit_from_word, edit_to_word):
@@ -431,7 +443,7 @@ def get_app():
431
 
432
  with gr.Row():
433
  with gr.Column(scale=2):
434
- input_audio = gr.Audio(value=f"{DEMO_PATH}/YOU1000000115_S0000252.wav", label="Input Audio", type="filepath", interactive=True)
435
  with gr.Group():
436
  original_transcript = gr.Textbox(label="Original transcript", lines=5, value=demo_original_transcript,
437
  info="Use whisperx model to get the transcript. Fix and align it if necessary.")
@@ -455,20 +467,20 @@ def get_app():
455
  mode = gr.Radio(label="Mode", choices=["TTS", "Edit", "Long TTS"], value="TTS")
456
  split_text = gr.Radio(label="Split text", choices=["Newline", "Sentence"], value="Newline",
457
  info="Split text into parts and run TTS for each part.", visible=False)
458
- edit_word_mode = gr.Radio(label="Edit word mode", choices=["Replace half", "Replace all"], value="Replace half",
459
  info="What to do with first and last word", visible=False)
460
 
461
  with gr.Group() as tts_mode_controls:
462
- prompt_to_word = gr.Dropdown(label="Last word in prompt", choices=demo_words, value=demo_words[12], interactive=True)
463
- prompt_end_time = gr.Slider(label="Prompt end time", minimum=0, maximum=7.479, step=0.001, value=3.689)
464
 
465
  with gr.Group(visible=False) as edit_mode_controls:
466
  with gr.Row():
467
- edit_from_word = gr.Dropdown(label="First word to edit", choices=demo_words, value=demo_words[9], interactive=True)
468
- edit_to_word = gr.Dropdown(label="Last word to edit", choices=demo_words, value=demo_words[12], interactive=True)
469
  with gr.Row():
470
- edit_start_time = gr.Slider(label="Edit from time", minimum=0, maximum=7.479, step=0.001, value=2.266)
471
- edit_end_time = gr.Slider(label="Edit to time", minimum=0, maximum=7.479, step=0.001, value=3.649)
472
 
473
  run_btn = gr.Button(value="Run")
474
 
@@ -487,7 +499,7 @@ def get_app():
487
  with gr.Accordion("Generation Parameters - change these if you are unhappy with the generation", open=False):
488
  stop_repetition = gr.Radio(label="stop_repetition", choices=[-1, 1, 2, 3, 4], value=3,
489
  info="if there are long silence in the generated audio, reduce the stop_repetition to 2 or 1. -1 = disabled")
490
- sample_batch_size = gr.Number(label="speech rate", value=2, precision=0,
491
  info="The higher the number, the faster the output will be. "
492
  "Under the hood, the model will generate this many samples and choose the shortest one. "
493
  "For giga330M_TTSEnhanced, 1 or 2 should be fine since the model is trained to do TTS.")
 
200
  buffer.seek(0)
201
  return buffer.read()
202
 
203
+ def replace_numbers_with_words(sentence):
204
+ sentence = re.sub(r'(\d+)', r' \1 ', sentence) # add spaces around numbers
205
+ def replace_with_words(match):
206
+ num = match.group(0)
207
+ try:
208
+ return num2words(num) # Convert numbers to words
209
+ except:
210
+ return num # In case num2words fails (unlikely with digits but just to be safe)
211
+ return re.sub(r'\b\d+\b', replace_with_words, sentence) # Regular expression that matches numbers
212
+
213
  @spaces.GPU(duration=90)
214
  def run(seed, left_margin, right_margin, codec_audio_sr, codec_sr, top_k, top_p, temperature,
215
  stop_repetition, sample_batch_size, kvcache, silence_tokens,
 
222
  raise gr.Error("Can't use smart transcript: whisper transcript not found")
223
 
224
  seed_everything(seed)
225
+ transcript = replace_numbers_with_words(transcript).replace(" ", " ").replace(" ", " ") # replace numbers with words, so that the phonemizer can do a better job
226
+
227
  if mode == "Long TTS":
228
  if split_text == "Newline":
229
  sentences = transcript.split('\n')
 
382
  - In Edit mode write full prompt</br>
383
  """
384
 
385
+ demo_original_transcript = "Gwynplaine had, besides, for his work and for his feats of strength, round his neck and over his shoulders, an esclavine of leather."
386
 
387
  demo_text = {
388
  "TTS": {
389
  "smart": "I cannot believe that the same model can also do text to speech synthesis too!",
390
+ "regular": "Gwynplaine had, besides, for his work and for his feats of strength, I cannot believe that the same model can also do text to speech synthesis too!"
391
  },
392
  "Edit": {
393
+ "smart": "take over the stage for half an hour,",
394
+ "regular": "Gwynplaine had, besides, for his work and for his feats of strength, take over the stage for half an hour, an esclavine of leather."
395
  },
396
  "Long TTS": {
397
  "smart": "You can run the model on a big text!\n"
398
  "Just write it line-by-line. Or sentence-by-sentence.\n"
399
  "If some sentences sound odd, just rerun the model on them, no need to generate the whole text again!",
400
+ "regular": "Gwynplaine had, besides, for his work and for his feats of strength, You can run the model on a big text!\n"
401
+ "Gwynplaine had, besides, for his work and for his feats of strength, Just write it line-by-line. Or sentence-by-sentence.\n"
402
+ "Gwynplaine had, besides, for his work and for his feats of strength, If some sentences sound odd, just rerun the model on them, no need to generate the whole text again!"
403
  }
404
  }
405
 
406
  all_demo_texts = {vv for k, v in demo_text.items() for kk, vv in v.items()}
407
 
408
+ demo_words = ['0.069 Gwynplain 0.611', '0.671 had, 0.912', '0.952 besides, 1.414', '1.494 for 1.634', '1.695 his 1.835', '1.915 work 2.136', '2.196 and 2.297', '2.337 for 2.517', '2.557 his 2.678', '2.758 feats 3.019', '3.079 of 3.139', '3.2 strength, 3.561', '4.022 round 4.263', '4.303 his 4.444', '4.524 neck 4.705', '4.745 and 4.825', '4.905 over 5.086', '5.146 his 5.266', '5.307 shoulders, 5.768', '6.23 an 6.33', '6.531 esclavine 7.133', '7.213 of 7.293', '7.353 leather. 7.614']
409
 
410
+ demo_words_info = [{'word': 'Gwynplain', 'start': 0.069, 'end': 0.611, 'score': 0.833}, {'word': 'had,', 'start': 0.671, 'end': 0.912, 'score': 0.879}, {'word': 'besides,', 'start': 0.952, 'end': 1.414, 'score': 0.863}, {'word': 'for', 'start': 1.494, 'end': 1.634, 'score': 0.89}, {'word': 'his', 'start': 1.695, 'end': 1.835, 'score': 0.669}, {'word': 'work', 'start': 1.915, 'end': 2.136, 'score': 0.916}, {'word': 'and', 'start': 2.196, 'end': 2.297, 'score': 0.766}, {'word': 'for', 'start': 2.337, 'end': 2.517, 'score': 0.808}, {'word': 'his', 'start': 2.557, 'end': 2.678, 'score': 0.786}, {'word': 'feats', 'start': 2.758, 'end': 3.019, 'score': 0.97}, {'word': 'of', 'start': 3.079, 'end': 3.139, 'score': 0.752}, {'word': 'strength,', 'start': 3.2, 'end': 3.561, 'score': 0.742}, {'word': 'round', 'start': 4.022, 'end': 4.263, 'score': 0.916}, {'word': 'his', 'start': 4.303, 'end': 4.444, 'score': 0.666}, {'word': 'neck', 'start': 4.524, 'end': 4.705, 'score': 0.908}, {'word': 'and', 'start': 4.745, 'end': 4.825, 'score': 0.882}, {'word': 'over', 'start': 4.905, 'end': 5.086, 'score': 0.847}, {'word': 'his', 'start': 5.146, 'end': 5.266, 'score': 0.791}, {'word': 'shoulders,', 'start': 5.307, 'end': 5.768, 'score': 0.729}, {'word': 'an', 'start': 6.23, 'end': 6.33, 'score': 0.854}, {'word': 'esclavine', 'start': 6.531, 'end': 7.133, 'score': 0.803}, {'word': 'of', 'start': 7.213, 'end': 7.293, 'score': 0.772}, {'word': 'leather.', 'start': 7.353, 'end': 7.614, 'score': 0.896}]
411
 
412
 
413
  def update_demo(mode, smart_transcript, edit_word_mode, transcript, edit_from_word, edit_to_word):
 
443
 
444
  with gr.Row():
445
  with gr.Column(scale=2):
446
+ input_audio = gr.Audio(value=f"{DEMO_PATH}/5895_34622_000026_000002.wav", label="Input Audio", type="filepath", interactive=True)
447
  with gr.Group():
448
  original_transcript = gr.Textbox(label="Original transcript", lines=5, value=demo_original_transcript,
449
  info="Use whisperx model to get the transcript. Fix and align it if necessary.")
 
467
  mode = gr.Radio(label="Mode", choices=["TTS", "Edit", "Long TTS"], value="TTS")
468
  split_text = gr.Radio(label="Split text", choices=["Newline", "Sentence"], value="Newline",
469
  info="Split text into parts and run TTS for each part.", visible=False)
470
+ edit_word_mode = gr.Radio(label="Edit word mode", choices=["Replace half", "Replace all"], value="Replace all",
471
  info="What to do with first and last word", visible=False)
472
 
473
  with gr.Group() as tts_mode_controls:
474
+ prompt_to_word = gr.Dropdown(label="Last word in prompt", choices=demo_words, value=demo_words[11], interactive=True)
475
+ prompt_end_time = gr.Slider(label="Prompt end time", minimum=0, maximum=7.614, step=0.001, value=3.600)
476
 
477
  with gr.Group(visible=False) as edit_mode_controls:
478
  with gr.Row():
479
+ edit_from_word = gr.Dropdown(label="First word to edit", choices=demo_words, value=demo_words[12], interactive=True)
480
+ edit_to_word = gr.Dropdown(label="Last word to edit", choices=demo_words, value=demo_words[18], interactive=True)
481
  with gr.Row():
482
+ edit_start_time = gr.Slider(label="Edit from time", minimum=0, maximum=7.614, step=0.001, value=4.022)
483
+ edit_end_time = gr.Slider(label="Edit to time", minimum=0, maximum=7.614, step=0.001, value=5.768)
484
 
485
  run_btn = gr.Button(value="Run")
486
 
 
499
  with gr.Accordion("Generation Parameters - change these if you are unhappy with the generation", open=False):
500
  stop_repetition = gr.Radio(label="stop_repetition", choices=[-1, 1, 2, 3, 4], value=3,
501
  info="if there are long silence in the generated audio, reduce the stop_repetition to 2 or 1. -1 = disabled")
502
+ sample_batch_size = gr.Number(label="speech rate", value=3, precision=0,
503
  info="The higher the number, the faster the output will be. "
504
  "Under the hood, the model will generate this many samples and choose the shortest one. "
505
  "For giga330M_TTSEnhanced, 1 or 2 should be fine since the model is trained to do TTS.")
demo/5895_34622_000026_000002.wav ADDED
Binary file (504 kB). View file
 
demo/YOU1000000115_S0000252.wav DELETED
Binary file (252 kB)
 
models/modules/__pycache__/__init__.cpython-39.pyc CHANGED
Binary files a/models/modules/__pycache__/__init__.cpython-39.pyc and b/models/modules/__pycache__/__init__.cpython-39.pyc differ
 
models/modules/__pycache__/activation.cpython-39.pyc CHANGED
Binary files a/models/modules/__pycache__/activation.cpython-39.pyc and b/models/modules/__pycache__/activation.cpython-39.pyc differ
 
models/modules/__pycache__/embedding.cpython-39.pyc CHANGED
Binary files a/models/modules/__pycache__/embedding.cpython-39.pyc and b/models/modules/__pycache__/embedding.cpython-39.pyc differ
 
models/modules/__pycache__/scaling.cpython-39.pyc CHANGED
Binary files a/models/modules/__pycache__/scaling.cpython-39.pyc and b/models/modules/__pycache__/scaling.cpython-39.pyc differ
 
models/modules/__pycache__/transformer.cpython-39.pyc CHANGED
Binary files a/models/modules/__pycache__/transformer.cpython-39.pyc and b/models/modules/__pycache__/transformer.cpython-39.pyc differ
 
models/modules/__pycache__/utils.cpython-39.pyc CHANGED
Binary files a/models/modules/__pycache__/utils.cpython-39.pyc and b/models/modules/__pycache__/utils.cpython-39.pyc differ
 
requirements.txt CHANGED
@@ -6,4 +6,5 @@ openai-whisper>=20231117
6
  spaces
7
  aeneas==1.7.3.0
8
  whisperx==3.1.1
9
- huggingface-hub==0.22.2
 
 
6
  spaces
7
  aeneas==1.7.3.0
8
  whisperx==3.1.1
9
+ huggingface-hub==0.22.2
10
+ num2words==0.5.13