import time import base64 import gradio as gr from sentence_transformers import SentenceTransformer import httpx import json import os import requests import urllib from os import path from pydub import AudioSegment MUBERT_LICENSE = os.environ.get('MUBERT_LICENSE') MUBERT_TOKEN = os.environ.get('MUBERT_TOKEN') #img_to_text = gr.Blocks.load(name="spaces/pharma/CLIP-Interrogator") img_to_text = gr.Blocks.load(name="spaces/fffiloni/CLIP-Interrogator-2") from share_btn import community_icon_html, loading_icon_html, share_js from utils import get_tags_for_prompts, get_mubert_tags_embeddings minilm = SentenceTransformer('all-MiniLM-L6-v2') mubert_tags_embeddings = get_mubert_tags_embeddings(minilm) def get_prompts(uploaded_image, track_duration, gen_intensity, gen_mode): print("calling clip interrogator") #prompt = img_to_text(uploaded_image, "ViT-L (best for Stable Diffusion 1.*)", "fast", fn_index=1)[0] prompt = img_to_text(uploaded_image, 'fast', 4, fn_index=1)[0] print(prompt) pat = get_pat_token() #music_result = get_music(pat, prompt, track_duration, gen_intensity, gen_mode) music_result = generate_track_by_prompt(pat, prompt, track_duration, gen_intensity, gen_mode) #print(pat) return music_result, gr.update(visible=True), gr.update(visible=True), gr.update(visible=True) def get_pat_token(): r = httpx.post('https://api-b2b.mubert.com/v2/GetServiceAccess', json={ "method": "GetServiceAccess", "params": { "email":"mail@mail.com", "phone":"+11234567890", "license": MUBERT_LICENSE, "token": MUBERT_TOKEN, } }) rdata = json.loads(r.text) #print(rdata) #assert rdata['status'] == 1, "probably incorrect e-mail" #pat = rdata['data']['pat'] print(rdata['data']['pat']) return rdata['data']['pat'] def get_music(pat, prompt, track_duration, gen_intensity, gen_mode): r = httpx.post('https://api-b2b.mubert.com/v2/TTMRecordTrack', json={ "method": "TTMRecordTrack", "params": { "text":"jazz music", "pat": pat, "mode":"track", "duration":track_duration, } }) rdata = json.loads(r.text) print(rdata['data']['tasks'][0]['download_link']) #assert rdata['status'] == 1, "probably incorrect e-mail" #track = rdata['data']['tasks']['download_link'] #print(track) return "done" def get_track_by_tags(tags, pat, duration, gen_intensity, gen_mode, maxit=20): r = httpx.post('https://api-b2b.mubert.com/v2/RecordTrackTTM', json={ "method": "RecordTrackTTM", "params": { "pat": pat, "duration": duration, "format": "wav", "intensity":gen_intensity, "tags": tags, "mode": gen_mode } }) rdata = json.loads(r.text) print(rdata) #assert rdata['status'] == 1, rdata['error']['text'] trackurl = rdata['data']['tasks'][0] print('Generating track ', end='') for i in range(maxit): r = httpx.get(trackurl) if r.status_code == 200: return trackurl time.sleep(1) def generate_track_by_prompt(pat, prompt, duration, gen_intensity, gen_mode): try: _, tags = get_tags_for_prompts(minilm, mubert_tags_embeddings, [prompt])[0] result = get_track_by_tags(tags, pat, int(duration), gen_intensity,i gen_mode) print(result) return result, ",".join(tags), "Success" except Exception as e: return None, "", str(e) def convert_mp3_to_wav(mp3_filepath): url = mp3_filepath save_as = "file.mp3" data = urllib.request.urlopen(url) f = open(save_as,'wb') f.write(data.read()) f.close() wave_file="file.wav" sound = AudioSegment.from_mp3(save_as) sound.export(wave_file, format="wav") return wave_file article = """

Follow Sylvain Filoni for future updates 🤗

You may also like:

""" with gr.Blocks(css="style.css") as demo: with gr.Column(elem_id="col-container"): gr.HTML("""

Image to Music

Sends an image in to CLIP Interrogator to generate a text prompt which is then run through Mubert text-to-music to generate music from the input image!

""") input_img = gr.Image(type="filepath", elem_id="input-img") music_output = gr.Audio(label="Result", type="filepath", elem_id="music-output").style(height="5rem") text_status = gr.Textbox(label="status") with gr.Group(elem_id="share-btn-container"): community_icon = gr.HTML(community_icon_html, visible=False) loading_icon = gr.HTML(loading_icon_html, visible=False) share_button = gr.Button("Share to community", elem_id="share-btn", visible=False) with gr.Accordion(label="Music Generation Options", open=False): track_duration = gr.Slider(minimum=20, maximum=120, value=30, step=5, label="Track duration", elem_id="duration-inp") with gr.Row(): gen_intensity = gr.Dropdown(choices=["low", "medium", "high"], value="medium", label="Intensity") gen_mode = gr.Radio(label="mode", choices=["track", "loop"], value="track") generate = gr.Button("Generate Music from Image") gr.HTML(article) generate.click(get_prompts, inputs=[input_img,track_duration,gen_intensity,gen_mode], outputs=[text_status, share_button, community_icon, loading_icon], api_name="i2m") share_button.click(None, [], [], _js=share_js) demo.queue(max_size=32, concurrency_count=20).launch()