Update app.py

app.py

@@ -1,33 +1,37 @@
-import os
 import torch
 import gradio as gr
 import torchaudio
 import time
 from datetime import datetime
+from transformers import pipeline
 from tortoise.api import TextToSpeech
 from tortoise.utils.text import split_and_recombine_text
 from tortoise.utils.audio import load_audio, load_voice, load_voices
 
-VOICE_OPTIONS = [
-    "kasuri",
-    "shanuka",
-    "indian_f_1",
-    "kushan",
-    "afshak",
-]
+# STT Setup
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
 
-def inference(
-    text,
-    voice
-):
-    if text is None or text.strip() == "":
-        raise gr.Error("Please provide text.")
+model_id = "openai/whisper-tiny"
+pipe = pipeline("automatic-speech-recognition", model=model_id, device=0)
 
-    texts = split_and_recombine_text(text)
+# TTS Setup
+VOICE_OPTIONS = ["indian_F_1", "indian_F_2", "indian_F_3", "indian_M_1", "indian_M_2", "indian_M_3"]
+tts = TextToSpeech(kv_cache=True, use_deepspeed=True, half=True)
 
-    voice_samples, conditioning_latents = load_voice(voice)
+def combined_inference(filepath, voice):
+    # STT: Convert audio to text
+    output = pipe(
+        filepath,
+        max_new_tokens=256,
+        generate_kwargs={"task": "transcribe", "language": "english"},
+        chunk_length_s=15,
+        batch_size=16,
+    )
+    text = output["text"]
 
-    start_time = time.time()
+    # TTS: Convert text back to audio
+    texts = split_and_recombine_text(text)
+    voice_samples, conditioning_latents = load_voice(voice)
 
     for j, text in enumerate(texts):
         for audio_frame in tts.tts_with_preset(
@@ -40,39 +44,29 @@ def inference(
             yield (24000, audio_frame.cpu().detach().numpy())
 
 def main():
-    title = "Tortoise TTS"
-    description = """
-    
-    """
-    text = gr.Textbox(
-        lines=4,
-        label="Text:",
-    )
+    title = "Combined STT and TTS"
+    description = ""
 
     voice = gr.Dropdown(
-        VOICE_OPTIONS, value="kasuri", label="Select voice:", type="value"
+        VOICE_OPTIONS, value="indian_F_1", label="Select voice:", type="value"
     )
 
-    output_audio = gr.Audio(label="streaming audio:", streaming=True, autoplay=True)
-
     interface = gr.Interface(
-        fn=inference,
+        fn=combined_inference,
         inputs=[
-            text,
+            gr.Audio(source="upload", type="filepath"),
             voice
         ],
         title=title,
         description=description,
-        outputs=[output_audio],
+        outputs=[gr.Audio(label="streaming audio:", streaming=True, autoplay=True)],
     )
     interface.queue().launch()
 
 if __name__ == "__main__":
-    tts = TextToSpeech(kv_cache=True, use_deepspeed=True, half=True)
-
-    with open("Tortoise_TTS_Runs_Scripts.log", "a") as f:
+    with open("Combined_STT_TTS_Runs_Scripts.log", "a") as f:
         f.write(
-            f"\n\n-------------------------Tortoise TTS Scripts Logs, {datetime.now()}-------------------------\n"
+            f"\n\n-------------------------Combined STT TTS Scripts Logs, {datetime.now()}-------------------------\n"
         )
 
-    main()
+    main()