Update app.py

app.py

@@ -1,9 +1,10 @@
 import gradio as gr
 import langcodes
-from transformers import pipeline
+from transformers import pipeline, VitsModel, AutoTokenizer, set_seed
 from huggingface_hub import InferenceClient
 from langdetect import detect, DetectorFactory
-# from IPython.display import Audio as IPythonAudio
+import uuid
+import scipy.io.wavfile as wav
 
 
 playground = gr.Blocks()
@@ -12,18 +13,19 @@ client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 image_pipe = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base")
 summary_pipe = pipeline("summarization", model="sshleifer/distilbart-cnn-12-6")
 ner_pipe = pipeline("ner", model="dslim/bert-base-NER")
-# narrator = pipeline("text-to-speech", model="kakao-enterprise/vits-ljs")
-
-# def generate_audio(text):
-#     # Generate speech from text
-#     narrated_text = narrator(text)
-#     audio_data = narrated_text["audio"][0]
-#     sampling_rate = narrated_text["sampling_rate"]
+tts_model = VitsModel.from_pretrained("facebook/mms-tts-eng")
+tts_tokenizer = AutoTokenizer.from_pretrained("facebook/mms-tts-eng")
+
+def gen_speech(text):
+    set_seed(555)  # Make it deterministic
+    input_text = tts_tokenizer(text, return_tensors="pt")
+    with torch.no_grad():
+        outputs = tts_model(**input_text)
+    waveform_np = outputs.waveform[0].cpu().numpy()
+    output_file = f"{str(uuid.uuid4())}.wav"
+    wav.write(output_file, rate=tts_model.config.sampling_rate, data=waveform_np)
+    return output_file
     
-#     # Use IPythonAudio to play the audio
-#     audio = IPythonAudio(audio_data, rate=sampling_rate)
-#     return audio_data, sampling_rate
-
 def detect_language(text):
     DetectorFactory.seed = 0  # Ensure consistent results
     return detect(text)
@@ -64,7 +66,8 @@ def respond(message, history: list[tuple[str, str]], system_message, max_tokens,
 def launch_image_pipe(input):
     out = image_pipe(input)
     text = out[0]['generated_text']
-    return text
+    audio_output_filepath = gen_speech(text)
+    return text, audio_output_filepath
 
 def translate(input_text, source, target):
     try:
@@ -139,10 +142,10 @@ with playground:
                 with gr.Column():
                     generated_textbox = gr.Textbox(lines=2, placeholder="", label="Generated Text")
                     # generate_audio_button = gr.Button(value="Generate Audio", variant="primary")
-                    # audio_output = gr.Audio(label="Generated Audio")
+                    audio_output = gr.Audio(type="filepath", label="Generated Speech")
                     ITT_Clear_button = gr.ClearButton(components=[img, generated_textbox], value="Clear")
                         
-            ITT_button.click(launch_image_pipe, inputs=[img], outputs=[generated_textbox])
+            ITT_button.click(launch_image_pipe, inputs=[img], outputs=[generated_textbox, audio_output])
             # generate_audio_button.click(generate_audio, inputs=[generated_textbox], outputs=[audio_output])
         
         ## ================================================================================================================================