Update app.py

app.py

@@ -5,13 +5,23 @@ import torch
 from spectro import wav_bytes_from_spectrogram_image
 from diffusers import StableDiffusionPipeline
 
+from transformers import BlipForConditionalGeneration, BlipProcessor
+
 from share_btn import community_icon_html, loading_icon_html, share_js
 
 model_id = "riffusion/riffusion-model-v1"
+blip_model_id = "Salesforce/blip-image-captioning-base"
 pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16)
-pipe = pipe.to("cuda")
+# pipe = pipe.to("cuda")
+
+blip_model = BlipForConditionalGeneration.from_pretrained(blip_model_id)
+processor = BlipProcessor.from_pretrained(blip_model_id)
 
-def predict(prompt):
+def predict(image):
+    inputs = processor(image)
+    output_blip = blip_model.generate(**inputs)
+    prompt = processor.decode(output_blip[0], skip_special_tokens=True)
+    
     spec = pipe(prompt).images[0]
     print(spec)
     wav = wav_bytes_from_spectrogram_image(spec)
@@ -131,7 +141,8 @@ with gr.Blocks(css=css) as demo:
         
         gr.HTML(title)
         
-        prompt_input = gr.Textbox(placeholder="a cat diva singing in a New York jazz club", label="Musical prompt", elem_id="prompt-in")
+        # prompt_input = gr.Textbox(placeholder="a cat diva singing in a New York jazz club", label="Musical prompt", elem_id="prompt-in")
+        image_input = gr.Image()
         send_btn = gr.Button(value="Get a new spectrogram ! ", elem_id="submit-btn")
             
     with gr.Column(elem_id="col-container-2"):