app.py

import gradio as gr
import torch
import os
from torch import autocast
from diffusers import StableDiffusionPipeline
from moviepy.editor import AudioFileClip, ImageClip
from pathlib import Path


output_dir = Path("temp/")
output_dir.mkdir(exist_ok=True, parents=True)
os.chdir(output_dir)


def process_inputs(prompt, audio):
    image = get_stable_diffusion_image(prompt)
    video = add_static_image_to_audio(image, audio)
    return video


def add_static_image_to_audio(image, audio):
    """Create and save a video file to `output_path` after
    combining a static image that is located in `image_path`
    with an audio file in `audio_path`"""
    # create the audio clip object
    audio_clip = AudioFileClip(audio)
    # create the image clip object
    image_clip = ImageClip(image)
    # use set_audio method from image clip to combine the audio with the image
    video_clip = image_clip.set_audio(audio_clip)
    # specify the duration of the new clip to be the duration of the audio clip
    video_clip.duration = audio_clip.duration
    # set the FPS to 1
    video_clip.fps = 1
    # write the resuling video clip
    path = "out.mp4"
    video_clip.write_videofile(path)
    return path


def get_stable_diffusion_image(prompt):
    path = "temp/image_out.png"
    stable_diffusion = gr.Blocks.load(name="spaces/stabilityai/stable-diffusion")
    gallery_dir = stable_diffusion(prompt, fn_index=2)
    # Rename gallery dir to sdout
    return [os.path.join(gallery_dir, img) for img in os.listdir(gallery_dir)][0]


iface = gr.Interface(
    fn=process_inputs, inputs=["text", gr.Audio(type="filepath")], outputs="video"
)
iface.launch()