import gradio as gr import torch from PIL import Image import numpy as np from spectro import wav_bytes_from_spectrogram_image from diffusers import StableDiffusionPipeline from diffusers import StableDiffusionImg2ImgPipeline #from share_btn import community_icon_html, loading_icon_html, share_js device = "cuda" MODEL_ID = "riffusion/riffusion-model-v1" pipe = StableDiffusionPipeline.from_pretrained(MODEL_ID, torch_dtype=torch.float16) pipe = pipe.to(device) pipe2 = StableDiffusionImg2ImgPipeline.from_pretrained(MODEL_ID, torch_dtype=torch.float16) pipe2 = pipe2.to(device) spectro_from_wav = gr.Interface.load("spaces/fffiloni/audio-to-spectrogram") def predict(prompt, negative_prompt, audio_input, duration): if audio_input == None : return classic(prompt, negative_prompt, duration) else : return style_transfer(prompt, negative_prompt, audio_input) def classic(prompt, negative_prompt, duration): if duration == 5: width_duration=512 else : width_duration = 512 + ((int(duration)-5) * 128) spec = pipe(prompt, negative_prompt=negative_prompt, height=512, width=width_duration).images[0] print(spec) wav = wav_bytes_from_spectrogram_image(spec) with open("output.wav", "wb") as f: f.write(wav[0].getbuffer()) return spec, 'output.wav' def style_transfer(prompt, negative_prompt, audio_input): spec = spectro_from_wav(audio_input) print(spec) # Open the image im = Image.open(spec) # Open the image im = image_from_spectrogram(im, 1) new_spectro = pipe2(prompt=prompt, image=im, strength=0.5, guidance_scale=7).images wav = wav_bytes_from_spectrogram_image(new_spectro[0]) with open("output.wav", "wb") as f: f.write(wav[0].getbuffer()) return new_spectro[0], 'output.wav' def image_from_spectrogram( spectrogram: np.ndarray, max_volume: float = 50, power_for_image: float = 0.25 ) -> Image.Image: """ Compute a spectrogram image from a spectrogram magnitude array. """ # Apply the power curve data = np.power(spectrogram, power_for_image) # Rescale to 0-255 data = data * 255 / max_volume # Invert data = 255 - data # Convert to a PIL image image = Image.fromarray(data.astype(np.uint8)) # Flip Y image = image.transpose(Image.FLIP_TOP_BOTTOM) # Convert to RGB image = image.convert("RGB") return image title = """

Riffusion real-time music generation

Describe a musical prompt, generate music by getting a spectrogram image & sound.

""" article = """

Do you need faster results ? You can skip the queue by duplicating this space: Duplicate Space

About the model: Riffusion is a latent text-to-image diffusion model capable of generating spectrogram images given any text input. These spectrograms can be converted into audio clips.

The Riffusion model was created by fine-tuning the Stable-Diffusion-v1-5 checkpoint.

The model is intended for research purposes only. Possible research areas and tasks include generation of artworks, audio, and use in creative processes, applications in educational or creative tools, research on generative models.

You may also like:

""" css = ''' #col-container, #col-container-2 {max-width: 510px; margin-left: auto; margin-right: auto;} a {text-decoration-line: underline; font-weight: 600;} div#record_btn > .mt-6 { margin-top: 0!important; } div#record_btn > .mt-6 button { width: 100%; height: 40px; } .footer { margin-bottom: 45px; margin-top: 10px; text-align: center; border-bottom: 1px solid #e5e5e5; } .footer>p { font-size: .8rem; display: inline-block; padding: 0 10px; transform: translateY(10px); background: white; } .dark .footer { border-color: #303030; } .dark .footer>p { background: #0b0f19; } .animate-spin { animation: spin 1s linear infinite; } @keyframes spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } } #share-btn-container { display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; width: 13rem; } #share-btn { all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.25rem !important; padding-bottom: 0.25rem !important;right:0; } #share-btn * { all: unset; } #share-btn-container div:nth-child(-n+2){ width: auto !important; min-height: 0px !important; } #share-btn-container .wrap { display: none !important; } ''' with gr.Blocks(css="style.css") as demo: with gr.Column(elem_id="col-container"): gr.HTML(title) prompt_input = gr.Textbox(placeholder="a cat diva singing in a New York jazz club", label="Musical prompt", elem_id="prompt-in") audio_input = gr.Audio(sources=["upload"], type="filepath", visible=False) with gr.Row(): negative_prompt = gr.Textbox(label="Negative prompt") duration_input = gr.Slider(label="Duration in seconds", minimum=5, maximum=10, step=1, value=8, elem_id="duration-slider") send_btn = gr.Button(value="Get a new spectrogram ! ", elem_id="submit-btn") with gr.Column(elem_id="col-container-2"): spectrogram_output = gr.Image(label="spectrogram image result", elem_id="img-out") sound_output = gr.Audio(type='filepath', label="spectrogram sound", elem_id="music-out") #with gr.Group(elem_id="share-btn-container"): # community_icon = gr.HTML(community_icon_html, visible=False) # loading_icon = gr.HTML(loading_icon_html, visible=False) # share_button = gr.Button("Share to community", elem_id="share-btn", visible=False) gr.HTML(article) send_btn.click(predict, inputs=[prompt_input, negative_prompt, audio_input, duration_input], outputs=[spectrogram_output, sound_output]) #share_button.click(None, [], [], _js=share_js) demo.queue(max_size=250).launch(debug=True)