Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import torch | |
| from spectro import wav_bytes_from_spectrogram_image | |
| from diffusers import StableDiffusionPipeline | |
| from transformers import BlipForConditionalGeneration, BlipProcessor | |
| from share_btn import community_icon_html, loading_icon_html, share_js | |
| model_id = "riffusion/riffusion-model-v1" | |
| blip_model_id = "Salesforce/blip-image-captioning-base" | |
| pipe = StableDiffusionPipeline.from_pretrained(model_id) | |
| pipe = pipe.to("cuda") | |
| blip_model = BlipForConditionalGeneration.from_pretrained(blip_model_id, torch_dtype=torch.float16).to("cuda") | |
| processor = BlipProcessor.from_pretrained(blip_model_id) | |
| def predict(image): | |
| inputs = processor(image, return_tensors="pt").to("cuda", torch.float16) | |
| output_blip = blip_model.generate(**inputs) | |
| prompt = processor.decode(output_blip[0], skip_special_tokens=True) | |
| spec = pipe(prompt).images[0] | |
| print(spec) | |
| wav = wav_bytes_from_spectrogram_image(spec) | |
| with open("output.wav", "wb") as f: | |
| f.write(wav[0].getbuffer()) | |
| return spec, 'output.wav', gr.update(visible=True), gr.update(visible=True), gr.update(visible=True) | |
| title = """ | |
| <div style="text-align: center; max-width: 500px; margin: 0 auto;"> | |
| <div | |
| style=" | |
| display: inline-flex; | |
| align-items: center; | |
| gap: 0.8rem; | |
| font-size: 1.75rem; | |
| margin-bottom: 10px; | |
| " | |
| > | |
| <h1 style="font-weight: 600; margin-bottom: 7px;"> | |
| Riffusion real-time image-to-music generation | |
| </h1> | |
| </div> | |
| <p style="margin-bottom: 10px;font-size: 94%;font-weight: 100;line-height: 1.5em;"> | |
| Describe a musical prompt, generate music by getting a spectrogram image & sound. | |
| </div> | |
| """ | |
| article = """ | |
| <p style="font-size: 0.8em;line-height: 1.2em;border: 1px solid #374151;border-radius: 8px;padding: 20px;"> | |
| About the model: Riffusion is a latent text-to-image diffusion model capable of generating spectrogram images given any text input. These spectrograms can be converted into audio clips. | |
| <br />— | |
| <br />The Riffusion model was created by fine-tuning the Stable-Diffusion-v1-5 checkpoint. | |
| <br />— | |
| <br />The model is intended for research purposes only. Possible research areas and tasks include | |
| generation of artworks, audio, and use in creative processes, applications in educational or creative tools, research on generative models. | |
| </p> | |
| <div class="footer"> | |
| <p> | |
| <a href="https://huggingface.co/riffusion/riffusion-model-v1" target="_blank">Riffusion model</a> by Seth Forsgren and Hayk Martiros - | |
| <a href="https://github.com/salesforce/BLIP" target="_blank"> BLIP Model </a> by Junnan Li et al. - Demo forked from 🤗 <a href="https://twitter.com/fffiloni" target="_blank">Sylvain Filoni</a>'s demo | |
| </p> | |
| </div> | |
| <p style="text-align: center;font-size: 94%"> | |
| Do you need faster results ? You can skip the queue by duplicating this space: | |
| <span style="display: flex;align-items: center;justify-content: center;height: 30px;"> | |
| <a href="https://huggingface.co/fffiloni/spectrogram-to-music?duplicate=true"><img src="https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14" alt="Duplicate Space"></a> | |
| </span> | |
| </p> | |
| """ | |
| css = ''' | |
| #col-container, #col-container-2 {max-width: 510px; margin-left: auto; margin-right: auto;} | |
| a {text-decoration-line: underline; font-weight: 600;} | |
| div#record_btn > .mt-6 { | |
| margin-top: 0!important; | |
| } | |
| div#record_btn > .mt-6 button { | |
| width: 100%; | |
| height: 40px; | |
| } | |
| .footer { | |
| margin-bottom: 45px; | |
| margin-top: 10px; | |
| text-align: center; | |
| border-bottom: 1px solid #e5e5e5; | |
| } | |
| .footer>p { | |
| font-size: .8rem; | |
| display: inline-block; | |
| padding: 0 10px; | |
| transform: translateY(10px); | |
| background: white; | |
| } | |
| .dark .footer { | |
| border-color: #303030; | |
| } | |
| .dark .footer>p { | |
| background: #0b0f19; | |
| } | |
| .animate-spin { | |
| animation: spin 1s linear infinite; | |
| } | |
| @keyframes spin { | |
| from { | |
| transform: rotate(0deg); | |
| } | |
| to { | |
| transform: rotate(360deg); | |
| } | |
| } | |
| #share-btn-container { | |
| display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; width: 13rem; | |
| } | |
| #share-btn { | |
| all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.25rem !important; padding-bottom: 0.25rem !important;right:0; | |
| } | |
| #share-btn * { | |
| all: unset; | |
| } | |
| #share-btn-container div:nth-child(-n+2){ | |
| width: auto !important; | |
| min-height: 0px !important; | |
| } | |
| #share-btn-container .wrap { | |
| display: none !important; | |
| } | |
| ''' | |
| with gr.Blocks(css=css) as demo: | |
| with gr.Column(elem_id="col-container"): | |
| gr.HTML(title) | |
| # prompt_input = gr.Textbox(placeholder="a cat diva singing in a New York jazz club", label="Musical prompt", elem_id="prompt-in") | |
| image_input = gr.Image() | |
| send_btn = gr.Button(value="Get a new spectrogram ! ", elem_id="submit-btn") | |
| with gr.Column(elem_id="col-container-2"): | |
| spectrogram_output = gr.Image(label="spectrogram image result", elem_id="img-out") | |
| sound_output = gr.Audio(type='filepath', label="spectrogram sound", elem_id="music-out") | |
| with gr.Group(elem_id="share-btn-container"): | |
| community_icon = gr.HTML(community_icon_html, visible=False) | |
| loading_icon = gr.HTML(loading_icon_html, visible=False) | |
| share_button = gr.Button("Share to community", elem_id="share-btn", visible=False) | |
| gr.HTML(article) | |
| send_btn.click(predict, inputs=[image_input], outputs=[spectrogram_output, sound_output, share_button, community_icon, loading_icon]) | |
| share_button.click(None, [], [], _js=share_js) | |
| demo.queue(max_size=250).launch(debug=True) | |