Spaces:
Running
on
A10G
Running
on
A10G
| import gradio as gr | |
| import torch | |
| from diffusers import AudioLDM2Pipeline | |
| # make Space compatible with CPU duplicates | |
| if torch.cuda.is_available(): | |
| device = "cuda" | |
| torch_dtype = torch.float16 | |
| else: | |
| device = "cpu" | |
| torch_dtype = torch.float32 | |
| # load the diffusers pipeline | |
| repo_id = "cvssp/audioldm2" | |
| pipe = AudioLDM2Pipeline.from_pretrained(repo_id, torch_dtype=torch_dtype).to(device) | |
| # pipe.unet = torch.compile(pipe.unet) | |
| # set the generator for reproducibility | |
| generator = torch.Generator(device) | |
| def text2audio(text, negative_prompt, duration, guidance_scale, random_seed, n_candidates): | |
| if text is None: | |
| raise gr.Error("Please provide a text input.") | |
| waveforms = pipe( | |
| text, | |
| audio_length_in_s=duration, | |
| guidance_scale=guidance_scale, | |
| num_inference_steps=200, | |
| negative_prompt=negative_prompt, | |
| num_waveforms_per_prompt=n_candidates if n_candidates else 1, | |
| generator=generator.manual_seed(int(random_seed)), | |
| )["audios"] | |
| return gr.make_waveform((16000, waveforms[0]), bg_image="bg.png") | |
| iface = gr.Blocks() | |
| with iface: | |
| gr.HTML( | |
| """ | |
| <div style="text-align: center; max-width: 700px; margin: 0 auto;"> | |
| <div | |
| style=" | |
| display: inline-flex; align-items: center; gap: 0.8rem; font-size: 1.75rem; | |
| " | |
| > | |
| <h1 style="font-weight: 900; margin-bottom: 7px; line-height: normal;"> | |
| AudioLDM 2: A General Framework for Audio, Music, and Speech Generation | |
| </h1> | |
| </div> <p style="margin-bottom: 10px; font-size: 94%"> | |
| <a href="https://arxiv.org/abs/2308.05734">[Paper]</a> <a href="https://audioldm.github.io/audioldm2">[Project | |
| page]</a> <a href="https://huggingface.co/docs/diffusers/main/en/api/pipelines/audioldm2">[🧨 | |
| Diffusers]</a> | |
| </p> | |
| </div> | |
| """ | |
| ) | |
| gr.HTML("""This is the demo for AudioLDM 2, powered by 🧨 Diffusers. Demo uses the checkpoint <a | |
| href="https://huggingface.co/cvssp/audioldm2"> AudioLDM 2 base</a>. For faster inference without waiting in | |
| queue, you may duplicate the space and upgrade to a GPU in the settings.""") | |
| gr.DuplicateButton() | |
| with gr.Group(): | |
| textbox = gr.Textbox( | |
| value="The vibrant beat of Brazilian samba drums.", | |
| max_lines=1, | |
| label="Input text", | |
| info="Your text is important for the audio quality. Please ensure it is descriptive by using more adjectives.", | |
| elem_id="prompt-in", | |
| ) | |
| negative_textbox = gr.Textbox( | |
| value="Low quality.", | |
| max_lines=1, | |
| label="Negative prompt", | |
| info="Enter a negative prompt not to guide the audio generation. Selecting appropriate negative prompts can improve the audio quality significantly.", | |
| elem_id="prompt-in", | |
| ) | |
| with gr.Accordion("Click to modify detailed configurations", open=False): | |
| seed = gr.Number( | |
| value=45, | |
| label="Seed", | |
| info="Change this value (any integer number) will lead to a different generation result.", | |
| ) | |
| duration = gr.Slider(5, 15, value=10, step=2.5, label="Duration (seconds)") | |
| guidance_scale = gr.Slider( | |
| 0, | |
| 7, | |
| value=3.5, | |
| step=0.5, | |
| label="Guidance scale", | |
| info="Larger => better quality and relevancy to text; Smaller => better diversity", | |
| ) | |
| n_candidates = gr.Slider( | |
| 1, | |
| 5, | |
| value=3, | |
| step=1, | |
| label="Number waveforms to generate", | |
| info="Automatic quality control. This number control the number of candidates (e.g., generate three audios and choose the best to show you). A larger value usually lead to better quality with heavier computation", | |
| ) | |
| outputs = gr.Video(label="Output", elem_id="output-video") | |
| btn = gr.Button("Submit") | |
| btn.click( | |
| text2audio, | |
| inputs=[textbox, negative_textbox, duration, guidance_scale, seed, n_candidates], | |
| # inputs=[textbox, negative_textbox, 10, guidance_scale, seed, n_candidates], | |
| outputs=[outputs], | |
| ) | |
| gr.HTML( | |
| """ | |
| <div class="footer" style="text-align: center"> | |
| <p>Share your generations with the community by clicking the share icon at the top right the generated audio!</p> | |
| <p>Follow the latest update of AudioLDM 2 on our<a href="https://audioldm.github.io/audioldm2" | |
| style="text-decoration: underline;" target="_blank"> Github repo</a> </p> | |
| <p>Model by <a | |
| href="https://twitter.com/LiuHaohe" style="text-decoration: underline;" target="_blank">Haohe | |
| Liu</a>. Code and demo by 🤗 Hugging Face.</p> | |
| </div> | |
| """ | |
| ) | |
| gr.Examples( | |
| [ | |
| ["A hammer is hitting a wooden surface.", "Low quality.", 10, 3.5, 45, 3], | |
| ["A cat is meowing for attention.", "Low quality.", 10, 3.5, 45, 3], | |
| ["An excited crowd cheering at a sports game.", "Low quality.", 10, 3.5, 45, 3], | |
| ["Birds singing sweetly in a blooming garden.", "Low quality.", 10, 3.5, 45, 3], | |
| ["A modern synthesizer creating futuristic soundscapes.", "Low quality.", 10, 3.5, 45, 3], | |
| ["The vibrant beat of Brazilian samba drums.", "Low quality.", 10, 3.5, 45, 3], | |
| ], | |
| fn=text2audio, | |
| inputs=[textbox, negative_textbox, duration, guidance_scale, seed, n_candidates], | |
| outputs=[outputs], | |
| cache_examples=True, | |
| ) | |
| gr.HTML( | |
| """ | |
| <div class="acknowledgements"> <p>Essential Tricks for Enhancing the Quality of Your Generated | |
| Audio</p> | |
| <p>1. Try using more adjectives to describe your sound. For example: "A man is speaking | |
| clearly and slowly in a large room" is better than "A man is speaking".</p> | |
| <p>2. Try using different random seeds, which can significantly affect the quality of the generated | |
| output.</p> | |
| <p>3. It's better to use general terms like 'man' or 'woman' instead of specific names for individuals or | |
| abstract objects that humans may not be familiar with.</p> | |
| <p>4. Using a negative prompt to not guide the diffusion process can improve the | |
| audio quality significantly. Try using negative prompts like 'low quality'.</p> | |
| </div> | |
| """ | |
| ) | |
| with gr.Accordion("Additional information", open=False): | |
| gr.HTML( | |
| """ | |
| <div class="acknowledgments"> | |
| <p> We build the model with data from <a href="http://research.google.com/audioset/">AudioSet</a>, | |
| <a href="https://freesound.org/">Freesound</a> and <a | |
| href="https://sound-effects.bbcrewind.co.uk/">BBC Sound Effect library</a>. We share this demo | |
| based on the <a | |
| href="https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/375954/Research.pdf">UK | |
| copyright exception</a> of data for academic research. | |
| </p> | |
| </div> | |
| """ | |
| ) | |
| iface.queue(max_size=20).launch() | |