Update app.py

app.py

@@ -1,7 +1,7 @@
 import torch
 import torch.nn as nn
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer, WhisperProcessor, WhisperForConditionalGeneration
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 
 
 class _MLPVectorProjector(nn.Module):
@@ -28,9 +28,16 @@ phi2_text = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=T
 tokenizer_text = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
 
 ## Audio model
-processor_audio = WhisperProcessor.from_pretrained("openai/whisper-small")
-model_audio = WhisperForConditionalGeneration.from_pretrained("openai/whisper-small")
-model_audio.config.forced_decoder_ids = None
+model_name_audio = "openai/whisper-small"
+#processor_audio = WhisperProcessor.from_pretrained("openai/whisper-small")
+#model_audio = WhisperForConditionalGeneration.from_pretrained("openai/whisper-small")
+#model_audio.config.forced_decoder_ids = None
+pipe = pipeline(
+    task="automatic-speech-recognition",
+    model=model_name_audio,
+    chunk_length_s=30,
+    device="cpu",
+)
 
 ## image model
 
@@ -60,15 +67,21 @@ def imageMode(image, question):
     return "In progress"
 
 def audioMode(audio):
-    #print('---------type of audio--------------')
+    if audio is None:
+        raise gr.Error("No audio file submitted! Please upload or record an audio file before submitting your request.")
+    
+    print('---------type of audio--------------')
+    print(type(audio))
+    print(audio)
+    text = pipe(audio, batch_size=8, generate_kwargs={"task": "transcribe"}, return_timestamps=True)["text"]
     #sampling_rate = audio[0]
-    audio_array = audio[1]
+    #audio_array = audio[1]
     #print(sampling_rate)
     #print(audio_array)
-    input_features = processor_audio(audio_array, sampling_rate=16000, return_tensors="pt").input_features 
-    predicted_ids = model_audio.generate(input_features)
-    transcription = processor_audio.batch_decode(predicted_ids, skip_special_tokens=True)
-    return transcription[0]
+    #input_features = processor_audio(audio_array, sampling_rate=16000, return_tensors="pt").input_features 
+    #predicted_ids = model_audio.generate(input_features)
+    #transcription = processor_audio.batch_decode(predicted_ids, skip_special_tokens=True)
+    return text
 
 
 interface_title = "TSAI-ERA-V1 - Capstone - Multimodal GPT Demo"
@@ -89,7 +102,7 @@ with gr.Blocks() as demo:
         image_text_output = gr.Textbox(label="Answer")
         
     with gr.Tab("Audio mode"):
-        audio_input = gr.Audio()
+        audio_input = gr.Audio(type="filepath", optional=True)
         audio_button = gr.Button("Submit")
         audio_text_output = gr.Textbox(label="Chat GPT like text")