File size: 2,135 Bytes
0d33acd
 
08b59d4
0d33acd
 
 
 
 
 
09780d3
 
 
0d33acd
9153210
 
0d33acd
 
 
 
 
 
09780d3
59ea9bc
 
0d33acd
 
4055f04
0d33acd
 
0c21172
0d33acd
 
a59f564
 
fcae65f
 
a59f564
 
 
46f82f4
 
 
f5b40d5
0d33acd
 
 
1bcb95d
f5b40d5
 
f22b6e2
1bcb95d
 
 
 
 
 
0d33acd
 
46f82f4
0d33acd
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import gradio as gr
import torch
import whisper
from diffusers import DiffusionPipeline
from transformers import (
    WhisperForConditionalGeneration,
    WhisperProcessor,
)

import os
MY_SECRET_TOKEN=os.environ.get('HF_TOKEN_SD')

device = "cuda" if torch.cuda.is_available() else "cpu"
model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-small").to(device)
processor = WhisperProcessor.from_pretrained("openai/whisper-small")

diffuser_pipeline = DiffusionPipeline.from_pretrained(
    "CompVis/stable-diffusion-v1-4",
    custom_pipeline="speech_to_image_diffusion",
    speech_model=model,
    speech_processor=processor,
    use_auth_token=MY_SECRET_TOKEN,
    revision="fp16",
    torch_dtype=torch.float16,
)

diffuser_pipeline.enable_attention_slicing()
diffuser_pipeline = diffuser_pipeline.to(device)


#β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”
# GRADIO SETUP
title = "Speech to Diffusion β€’ Community Pipeline"
description = """
<p style='text-align: center;'>This demo can generate an image from an audio sample using pre-trained OpenAI whisper-small and Stable Diffusion.<br />
Community examples consist of both inference and training examples that have been added by the community.<br />
<a href='https://github.com/huggingface/diffusers/tree/main/examples/community#speech-to-image' target='_blank'> Click here for more information about community pipelines </a>
</p>
"""
article = """
<p style='text-align: center;'>Community pipeline by Mikail Duzenli β€’ Gradio demo by Sylvain Filoni & Ahsen Khaliq<p>
"""
audio_input = gr.Audio(source="microphone", type="filepath")
image_output = gr.Image()

def speech_to_text(audio_sample):
  
  process_audio = whisper.load_audio(audio_sample)
  output = diffuser_pipeline(process_audio)
 
  print(f"""
  β€”β€”β€”β€”β€”β€”β€”β€”
  output: {output}
  β€”β€”β€”β€”β€”β€”β€”β€”
  """)
  
  return output.images[0]

demo = gr.Interface(fn=speech_to_text, inputs=audio_input, outputs=image_output, title=title, description=description, article=article)
demo.launch()