app.py

import gradio as gr
from transformers import pipeline
# from TTS.api import TTS

import librosa
import numpy as np
import torch

from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan


checkpoint = "microsoft/speecht5_tts"
processor = SpeechT5Processor.from_pretrained(checkpoint)
model = SpeechT5ForTextToSpeech.from_pretrained(checkpoint)
vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")

def tts(text):
    if len(text.strip()) == 0:
        return (16000, np.zeros(0).astype(np.int16))

    inputs = processor(text=text, return_tensors="pt")

    # limit input length
    input_ids = inputs["input_ids"]
    input_ids = input_ids[..., :model.config.max_text_positions]

    # if speaker == "Surprise Me!":
    #     # load one of the provided speaker embeddings at random
    #     idx = np.random.randint(len(speaker_embeddings))
    #     key = list(speaker_embeddings.keys())[idx]
    #     speaker_embedding = np.load(speaker_embeddings[key])

    #     # randomly shuffle the elements
    #     np.random.shuffle(speaker_embedding)

    #     # randomly flip half the values
    #     x = (np.random.rand(512) >= 0.5) * 1.0
    #     x[x == 0] = -1.0
    #     speaker_embedding *= x

        #speaker_embedding = np.random.rand(512).astype(np.float32) * 0.3 - 0.15
    # else:
    speaker_embedding = np.load("cmu_us_bdl_arctic-wav-arctic_a0009.npy")

    speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0)

    speech = model.generate_speech(input_ids, speaker_embedding, vocoder=vocoder)

    speech = (speech.numpy() * 32767).astype(np.int16)
    return (16000, speech)


captioner = pipeline(model="microsoft/git-base")
# tts = TTS(model_name="tts_models/multilingual/multi-dataset/your_tts", progress_bar=False, gpu=False)


def predict(image):
    text = captioner(image)[0]["generated_text"]

    # audio_output = "output.wav"
    # tts.tts_to_file(text, speaker=tts.speakers[0], language="en", file_path=audio_output)
    audio = tts(text)
    
    return text, audio

# theme = gr.themes.Default(primary_hue="#002A5B")

demo = gr.Interface(
    fn=predict,
    inputs=gr.Image(type="pil",label="Environment"),
    outputs=[gr.Textbox(label="Caption"), gr.Audio(type="numpy",label="Audio Feedback")],
    css=".gradio-container {background-color: #002A5B}",
    theme=gr.themes.Soft()
)

demo.launch()

# gr.Interface.load("models/ronniet/git-base-env").launch()
# gr.Interface.load("models/microsoft/git-base").launch()