File size: 7,469 Bytes
ea270ed
984cfda
 
ea270ed
f3ecefe
984cfda
 
 
 
 
 
 
 
 
 
 
 
ea270ed
984cfda
 
ea270ed
 
984cfda
 
 
ea270ed
984cfda
 
 
ea270ed
984cfda
 
 
 
 
 
 
 
ea270ed
bf49b51
ea270ed
 
 
 
 
 
 
984cfda
ea270ed
 
 
 
 
984cfda
 
 
 
ea270ed
 
 
 
bf49b51
984cfda
bf49b51
 
984cfda
ea270ed
bf49b51
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ea270ed
1a5a4b3
bf49b51
 
 
 
 
 
 
 
 
 
 
 
 
 
 
984cfda
ea270ed
bf49b51
 
ea270ed
bf49b51
 
1a5a4b3
 
bf49b51
 
ea270ed
bf49b51
ea270ed
bf49b51
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ea270ed
 
bf49b51
 
 
 
 
 
 
 
 
 
ea270ed
 
984cfda
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
import gradio as gr
import torch
from diffusers import AudioLDM2Pipeline


# make Space compatible with CPU duplicates
if torch.cuda.is_available():
    device = "cuda"
    torch_dtype = torch.float16
else:
    device = "cpu"
    torch_dtype = torch.float32

# load the diffusers pipeline
repo_id = "cvssp/audioldm2"
pipe = AudioLDM2Pipeline.from_pretrained(repo_id, torch_dtype=torch_dtype).to(device)
# pipe.unet = torch.compile(pipe.unet)

# set the generator for reproducibility
generator = torch.Generator(device)


def text2audio(text, negative_prompt, duration, guidance_scale, random_seed, n_candidates):
    if text is None:
        raise gr.Error("Please provide a text input.")

    waveforms = pipe(
        text,
        audio_length_in_s=duration,
        guidance_scale=guidance_scale,
        num_inference_steps=200,
        negative_prompt=negative_prompt,
        num_waveforms_per_prompt=n_candidates if n_candidates else 1,
        generator=generator.manual_seed(int(random_seed)),
    )["audios"]

    return gr.make_waveform((16000, waveforms[0]), bg_image="bg.png")


iface = gr.Blocks()

with iface:
    gr.HTML(
        """
            <div style="text-align: center; max-width: 700px; margin: 0 auto;">
              <div
                style="
                  display: inline-flex; align-items: center; gap: 0.8rem; font-size: 1.75rem;
                "
              >
                <h1 style="font-weight: 900; margin-bottom: 7px; line-height: normal;">
                  AudioLDM 2: A General Framework for Audio, Music, and Speech Generation
                </h1>
              </div> <p style="margin-bottom: 10px; font-size: 94%">
                <a href="https://arxiv.org/abs/2308.05734">[Paper]</a> <a href="https://audioldm.github.io/audioldm2">[Project
                page]</a> <a href="https://huggingface.co/docs/diffusers/main/en/api/pipelines/audioldm2">[🧨
                Diffusers]</a>
              </p>
            </div>
        """
    )
    gr.HTML("""This is the demo for AudioLDM 2, powered by 🧨 Diffusers. Demo uses the checkpoint <a
        href="https://huggingface.co/cvssp/audioldm2"> AudioLDM 2 base</a>. For faster inference without waiting in
        queue, you may duplicate the space and upgrade to a GPU in the settings.""")
    gr.DuplicateButton()

    with gr.Group():
        textbox = gr.Textbox(
            value="The vibrant beat of Brazilian samba drums.",
            max_lines=1,
            label="Input text",
            info="Your text is important for the audio quality. Please ensure it is descriptive by using more adjectives.",
            elem_id="prompt-in",
        )
        negative_textbox = gr.Textbox(
            value="Low quality.",
            max_lines=1,
            label="Negative prompt",
            info="Enter a negative prompt not to guide the audio generation. Selecting appropriate negative prompts can improve the audio quality significantly.",
            elem_id="prompt-in",
        )

        with gr.Accordion("Click to modify detailed configurations", open=False):
            seed = gr.Number(
                value=45,
                label="Seed",
                info="Change this value (any integer number) will lead to a different generation result.",
            )
            duration = gr.Slider(5, 15, value=10, step=2.5, label="Duration (seconds)")
            guidance_scale = gr.Slider(
                0,
                7,
                value=3.5,
                step=0.5,
                label="Guidance scale",
                info="Larger => better quality and relevancy to text; Smaller => better diversity",
            )
            n_candidates = gr.Slider(
                1,
                5,
                value=3,
                step=1,
                label="Number waveforms to generate",
                info="Automatic quality control. This number control the number of candidates (e.g., generate three audios and choose the best to show you). A larger value usually lead to better quality with heavier computation",
            )

        outputs = gr.Video(label="Output", elem_id="output-video")
        btn = gr.Button("Submit")

    btn.click(
        text2audio,
        inputs=[textbox, negative_textbox, duration, guidance_scale, seed, n_candidates],
        # inputs=[textbox, negative_textbox, 10, guidance_scale, seed, n_candidates],
        outputs=[outputs],
    )

    gr.HTML(
        """
    <div class="footer" style="text-align: center">
        <p>Share your generations with the community by clicking the share icon at the top right the generated audio!</p>
        <p>Follow the latest update of AudioLDM 2 on our<a href="https://audioldm.github.io/audioldm2"
        style="text-decoration: underline;" target="_blank"> Github repo</a> </p> 
        <p>Model by <a
        href="https://twitter.com/LiuHaohe" style="text-decoration: underline;" target="_blank">Haohe
        Liu</a>. Code and demo by 🤗 Hugging Face.</p>
    </div>
    """
    )
    gr.Examples(
        [
            ["A hammer is hitting a wooden surface.", "Low quality.", 10, 3.5, 45, 3],
            ["A cat is meowing for attention.", "Low quality.", 10, 3.5, 45, 3],
            ["An excited crowd cheering at a sports game.", "Low quality.", 10, 3.5, 45, 3],
            ["Birds singing sweetly in a blooming garden.", "Low quality.", 10, 3.5, 45, 3],
            ["A modern synthesizer creating futuristic soundscapes.", "Low quality.", 10, 3.5, 45, 3],
            ["The vibrant beat of Brazilian samba drums.", "Low quality.", 10, 3.5, 45, 3],
        ],
        fn=text2audio,
        inputs=[textbox, negative_textbox, duration, guidance_scale, seed, n_candidates],
        outputs=[outputs],
        cache_examples=True,
    )
    gr.HTML(
        """
            <div class="acknowledgements"> <p>Essential Tricks for Enhancing the Quality of Your Generated
            Audio</p>
            <p>1. Try using more adjectives to describe your sound. For example: "A man is speaking
            clearly and slowly in a large room" is better than "A man is speaking".</p>
            <p>2. Try using different random seeds, which can significantly affect the quality of the generated 
            output.</p>
            <p>3. It's better to use general terms like 'man' or 'woman' instead of specific names for individuals or 
            abstract objects that humans may not be familiar with.</p>
            <p>4. Using a negative prompt to not guide the diffusion process can improve the
            audio quality significantly. Try using negative prompts like 'low quality'.</p>
            </div>
            """
    )
    with gr.Accordion("Additional information", open=False):
        gr.HTML(
            """
            <div class="acknowledgments">
                <p> We build the model with data from <a href="http://research.google.com/audioset/">AudioSet</a>,
                <a href="https://freesound.org/">Freesound</a> and <a
                href="https://sound-effects.bbcrewind.co.uk/">BBC Sound Effect library</a>. We share this demo
                based on the <a
                href="https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/375954/Research.pdf">UK
                copyright exception</a> of data for academic research. 
                </p>
            </div>
            """
        )

iface.queue(max_size=20).launch()