Spaces:
Running
on
T4
Running
on
T4
| import gradio as gr | |
| import torch | |
| from PIL import Image | |
| import numpy as np | |
| from spectro import wav_bytes_from_spectrogram_image | |
| from diffusers import StableDiffusionPipeline | |
| from diffusers import StableDiffusionImg2ImgPipeline | |
| from share_btn import community_icon_html, loading_icon_html, share_js | |
| device = "cuda" | |
| MODEL_ID = "riffusion/riffusion-model-v1" | |
| pipe = StableDiffusionPipeline.from_pretrained(MODEL_ID, torch_dtype=torch.float16) | |
| pipe = pipe.to(device) | |
| pipe2 = StableDiffusionImg2ImgPipeline.from_pretrained(MODEL_ID, torch_dtype=torch.float16) | |
| pipe2 = pipe2.to(device) | |
| spectro_from_wav = gr.Interface.load("spaces/fffiloni/audio-to-spectrogram") | |
| def predict(prompt, negative_prompt, audio_input, duration): | |
| if audio_input == None : | |
| return classic(prompt, negative_prompt, duration) | |
| else : | |
| return style_transfer(prompt, negative_prompt, audio_input) | |
| def classic(prompt, negative_prompt, duration): | |
| if duration == 5: | |
| width_duration=512 | |
| else : | |
| width_duration = 512 + ((int(duration)-5) * 128) | |
| spec = pipe(prompt, negative_prompt=negative_prompt, height=512, width=width_duration).images[0] | |
| print(spec) | |
| wav = wav_bytes_from_spectrogram_image(spec) | |
| with open("output.wav", "wb") as f: | |
| f.write(wav[0].getbuffer()) | |
| return spec, 'output.wav', gr.update(visible=True), gr.update(visible=True), gr.update(visible=True) | |
| def style_transfer(prompt, negative_prompt, audio_input): | |
| spec = spectro_from_wav(audio_input) | |
| print(spec) | |
| # Open the image | |
| im = Image.open(spec) | |
| # Open the image | |
| im = image_from_spectrogram(im, 1) | |
| new_spectro = pipe2(prompt=prompt, image=im, strength=0.5, guidance_scale=7).images | |
| wav = wav_bytes_from_spectrogram_image(new_spectro[0]) | |
| with open("output.wav", "wb") as f: | |
| f.write(wav[0].getbuffer()) | |
| return new_spectro[0], 'output.wav', gr.update(visible=True), gr.update(visible=True), gr.update(visible=True) | |
| def image_from_spectrogram( | |
| spectrogram: np.ndarray, max_volume: float = 50, power_for_image: float = 0.25 | |
| ) -> Image.Image: | |
| """ | |
| Compute a spectrogram image from a spectrogram magnitude array. | |
| """ | |
| # Apply the power curve | |
| data = np.power(spectrogram, power_for_image) | |
| # Rescale to 0-255 | |
| data = data * 255 / max_volume | |
| # Invert | |
| data = 255 - data | |
| # Convert to a PIL image | |
| image = Image.fromarray(data.astype(np.uint8)) | |
| # Flip Y | |
| image = image.transpose(Image.FLIP_TOP_BOTTOM) | |
| # Convert to RGB | |
| image = image.convert("RGB") | |
| return image | |
| title = """ | |
| <div style="text-align: center; max-width: 500px; margin: 0 auto;"> | |
| <div | |
| style=" | |
| display: inline-flex; | |
| align-items: center; | |
| gap: 0.8rem; | |
| font-size: 1.75rem; | |
| margin-bottom: 10px; | |
| line-height: 1em; | |
| " | |
| > | |
| <h1 style="font-weight: 600; margin-bottom: 7px;"> | |
| Riffusion real-time music generation | |
| </h1> | |
| </div> | |
| <p style="margin-bottom: 10px;font-size: 94%;font-weight: 100;line-height: 1.5em;"> | |
| Describe a musical prompt, generate music by getting a spectrogram image & sound. | |
| </p> | |
| </div> | |
| """ | |
| article = """ | |
| <p style="text-align: center;font-size: 94%;margin-bottom: 20px;"> | |
| Do you need faster results ? You can skip the queue by duplicating this space: | |
| <span style="display: flex;align-items: center;justify-content: center;height: 30px;"> | |
| <a style="margin-right: 10px;" href="https://huggingface.co/fffiloni/spectrogram-to-music?duplicate=true"><img src="https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14" alt="Duplicate Space"></a> | |
| <a href="https://colab.research.google.com/drive/1FhH3HlN8Ps_Pr9OR6Qcfbfz7utDvICl0?usp=sharing" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" /></a> | |
| </span> | |
| </p> | |
| <p style="font-size: 0.8em;line-height: 1.2em;border: 1px solid #374151;border-radius: 8px;padding: 20px;"> | |
| About the model: Riffusion is a latent text-to-image diffusion model capable of generating spectrogram images given any text input. These spectrograms can be converted into audio clips. | |
| <br />— | |
| <br />The Riffusion model was created by fine-tuning the Stable-Diffusion-v1-5 checkpoint. | |
| <br />— | |
| <br />The model is intended for research purposes only. Possible research areas and tasks include | |
| generation of artworks, audio, and use in creative processes, applications in educational or creative tools, research on generative models. | |
| </p> | |
| <div class="footer"> | |
| <p> | |
| <a href="https://huggingface.co/riffusion/riffusion-model-v1" target="_blank">Riffusion model</a> by Seth Forsgren and Hayk Martiros - | |
| Demo by 🤗 <a href="https://twitter.com/fffiloni" target="_blank">Sylvain Filoni</a> | |
| </p> | |
| </div> | |
| <div id="may-like-container" style="display: flex;justify-content: center;flex-direction: column;align-items: center;"> | |
| <p style="font-size: 0.8em;margin-bottom: 4px;">You may also like: </p> | |
| <div id="may-like" style="display:flex; align-items:center; justify-content: center;height:20px;"> | |
| <svg height="20" width="158" style="margin-left:4px"> | |
| <a href="https://huggingface.co/spaces/fffiloni/img-to-music" target="_blank"> | |
| <image href="https://img.shields.io/badge/🤗 Spaces-Image to Music-blue" src="https://img.shields.io/badge/🤗 Spaces-Image to Music-blue.png" height="20"/> | |
| </a> | |
| </svg> | |
| </div> | |
| </div> | |
| """ | |
| css = ''' | |
| #col-container, #col-container-2 {max-width: 510px; margin-left: auto; margin-right: auto;} | |
| a {text-decoration-line: underline; font-weight: 600;} | |
| div#record_btn > .mt-6 { | |
| margin-top: 0!important; | |
| } | |
| div#record_btn > .mt-6 button { | |
| width: 100%; | |
| height: 40px; | |
| } | |
| .footer { | |
| margin-bottom: 45px; | |
| margin-top: 10px; | |
| text-align: center; | |
| border-bottom: 1px solid #e5e5e5; | |
| } | |
| .footer>p { | |
| font-size: .8rem; | |
| display: inline-block; | |
| padding: 0 10px; | |
| transform: translateY(10px); | |
| background: white; | |
| } | |
| .dark .footer { | |
| border-color: #303030; | |
| } | |
| .dark .footer>p { | |
| background: #0b0f19; | |
| } | |
| .animate-spin { | |
| animation: spin 1s linear infinite; | |
| } | |
| @keyframes spin { | |
| from { | |
| transform: rotate(0deg); | |
| } | |
| to { | |
| transform: rotate(360deg); | |
| } | |
| } | |
| #share-btn-container { | |
| display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; width: 13rem; | |
| } | |
| #share-btn { | |
| all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.25rem !important; padding-bottom: 0.25rem !important;right:0; | |
| } | |
| #share-btn * { | |
| all: unset; | |
| } | |
| #share-btn-container div:nth-child(-n+2){ | |
| width: auto !important; | |
| min-height: 0px !important; | |
| } | |
| #share-btn-container .wrap { | |
| display: none !important; | |
| } | |
| ''' | |
| with gr.Blocks(css="style.css") as demo: | |
| with gr.Column(elem_id="col-container"): | |
| gr.HTML(title) | |
| prompt_input = gr.Textbox(placeholder="a cat diva singing in a New York jazz club", label="Musical prompt", elem_id="prompt-in") | |
| audio_input = gr.Audio(source="upload", type="filepath", visible=False) | |
| with gr.Row(): | |
| negative_prompt = gr.Textbox(label="Negative prompt") | |
| duration_input = gr.Slider(label="Duration in seconds", minimum=5, maximum=60, step=1, value=8, elem_id="duration-slider") | |
| send_btn = gr.Button(value="Get a new spectrogram ! ", elem_id="submit-btn") | |
| with gr.Column(elem_id="col-container-2"): | |
| spectrogram_output = gr.Image(label="spectrogram image result", elem_id="img-out") | |
| sound_output = gr.Audio(type='filepath', label="spectrogram sound", elem_id="music-out") | |
| with gr.Group(elem_id="share-btn-container"): | |
| community_icon = gr.HTML(community_icon_html, visible=False) | |
| loading_icon = gr.HTML(loading_icon_html, visible=False) | |
| share_button = gr.Button("Share to community", elem_id="share-btn", visible=False) | |
| gr.HTML(article) | |
| send_btn.click(predict, inputs=[prompt_input, negative_prompt, audio_input, duration_input], outputs=[spectrogram_output, sound_output, share_button, community_icon, loading_icon]) | |
| share_button.click(None, [], [], _js=share_js) | |
| demo.queue(max_size=250).launch(debug=True) | |