app update

.gitignore

@@ -33,3 +33,6 @@ gradio.egg-info/
 
 # Virtual Environment
 .env/
+
+#others
+test_inputs/

README.md

@@ -1,13 +1,4 @@
----
-title: Imagebind Demo
-emoji: 📉
-colorFrom: indigo
-colorTo: purple
-sdk: gradio
-sdk_version: 4.19.2
-app_file: app.py
-pinned: false
-license: mit
----
+# Imagebind demo
+
+A gradio app showcasing multi-modal capabilities of Imagebind supported via lanceDB API
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -2,166 +2,89 @@ import lancedb
 import lancedb.embeddings.imagebind
 from lancedb.embeddings import get_registry
 from lancedb.pydantic import LanceModel, Vector
-import pandas as pd
-import os
 import gradio as gr
+from downloader import dowload_and_save_audio, dowload_and_save_image, base_path
 
 model = get_registry().get("imagebind").create()
 
+
 class TextModel(LanceModel):
-   text: str 
-   image_uri: str = model.SourceField()
-   audio_path: str 
-   vector: Vector(model.ndims()) = model.VectorField()
-
-text_list=["A dragon", "A car", "A bird"]
-
-base_path = os.path.dirname(os.path.abspath(__file__))
-# image_paths=["/test_inputs/dragon.jpg",
-#               "/test_inputs/car.jpg",
-#               "/test_inputs/bird_image.jpg"]
-# audio_paths=["/test_inputs/dragon-growl-37570.wav",
-#              "/test_inputs/car_audio.wav",
-#               "/test_inputs/bird_audio.wav"]
-
-image_paths=["./test_inputs/dragon.jpg",
-              "./test_inputs/car.jpg",
-              "./test_inputs/bird_image.jpg"]
-audio_paths=["./test_inputs/dragon-growl-37570.wav",
-             "./test_inputs/car_audio.wav",
-              "./test_inputs/bird_audio.wav"]
+    text: str
+    image_uri: str = model.SourceField()
+    audio_path: str
+    vector: Vector(model.ndims()) = model.VectorField()
+
+
+text_list = ["A bird", "A dragon", "A car"]
+image_paths = dowload_and_save_image()
+audio_paths = dowload_and_save_audio()
+
 # Load data
 inputs = [
-    {
-        "text": a,
-        "audio_path":b,
-        "image_uri":c
-    } for a,
-    b,
-    c in zip(text_list,
-                       audio_paths,
-                       image_paths)
+    {"text": a, "audio_path": b, "image_uri": c}
+    for a, b, c in zip(text_list, audio_paths, image_paths)
 ]
 
-
 db = lancedb.connect("/tmp/lancedb")
-table = db.create_table("img_bind",schema=TextModel)
+table = db.create_table("img_bind", schema=TextModel)
 table.add(inputs)
 
 
 def process_image(inp_img) -> str:
-
     actual = (
-    table.search(inp_img, vector_column_name="vector")
-    .limit(1)
-    .to_pydantic(TextModel)[0]
+        table.search(inp_img, vector_column_name="vector")
+        .limit(1)
+        .to_pydantic(TextModel)[0]
     )
 
     return actual.text, actual.audio_path
 
-def process_text(inp_text) -> str:
 
+def process_text(inp_text) -> str:
     actual = (
-    table.search(inp_text, vector_column_name="vector")
-    .limit(1)
-    .to_pydantic(TextModel)[0]
+        table.search(inp_text, vector_column_name="vector")
+        .limit(1)
+        .to_pydantic(TextModel)[0]
     )
 
     return actual.image_uri, actual.audio_path
 
-def process_audio(inp_audio) -> str:
 
+def process_audio(inp_audio) -> str:
     actual = (
-    table.search(inp_audio, vector_column_name="vector")
-    .limit(1)
-    .to_pydantic(TextModel)[0]
+        table.search(inp_audio, vector_column_name="vector")
+        .limit(1)
+        .to_pydantic(TextModel)[0]
     )
 
     return actual.image_uri, actual.text
 
-css = """
-output-audio, output-text { 
-    display: none; 
-}
-img { 
-width: 300px; 
-height: 450px; 
-object-fit: cover; 
-
-"""
-with gr.Blocks(css=css) as app:
-    # Using Markdown for custom CSS (optional)
-    with gr.Tab("Image to Text and Audio"):
-        with gr.Row():
-            with gr.Column():
-
-                inp1 = gr.Image(value=image_paths[0],type='filepath',elem_id='img',interactive=False)
-                output_audio1 = gr.Audio(label="Output Audio", elem_id="output-audio")
-                output_text1 = gr.Textbox(label="Output Text", elem_id="output-text")
-                btn_img1 = gr.Button("Retrieve")
-                
-                # output_audio1 = gr.Audio(label="Output Audio 1", elem_id="output-audio1")
-            with gr.Column():
-                inp2 = gr.Image(value=image_paths[1],type='filepath',elem_id='img',interactive=False)
-                output_audio2 = gr.Audio(label="Output Audio", elem_id="output-audio")
-                output_text2 = gr.Textbox(label="Output Text", elem_id="output-text")
-                btn_img2 = gr.Button("Retrieve")
-            
-            with gr.Column():
-                inp3 = gr.Image(value=image_paths[2],type='filepath',elem_id='img',interactive=False)
-                output_audio3 = gr.Audio(label="Output Audio", elem_id="output-audio")
-                output_text3 = gr.Textbox(label="Output Text", elem_id="output-text")
-                btn_img3 = gr.Button("Retrieve")
-    
-    with gr.Tab("Text to Image and Audio"):
-        with gr.Row():
-            with gr.Column():
-                input_txt1 = gr.Textbox(label="Enter a prompt:", elem_id="output-text")
-                output_audio4 = gr.Audio(label="Output Audio", elem_id="output-audio")
-                output_img1 = gr.Image(type='filepath',elem_id='img')
-                
-            # with gr.Column():
-            #     input_txt2 = gr.Textbox(label="Enter a prompt:", elem_id="output-text")
-            #     output_audio5 = gr.Audio(label="Output Audio", elem_id="output-audio")
-            #     output_img2 = gr.Image(type='filepath',elem_id='img')
-             
-            
-            # with gr.Column():
-            #     input_txt3 = gr.Textbox(label="Enter a prompt:", elem_id="output-text")
-            #     output_audio6 = gr.Audio(label="Output Audio", elem_id="output-audio")
-            #     output_img3 = gr.Image(type='filepath',elem_id='img')
-
-    with gr.Tab("Audio to Image and Text"):
-        with gr.Row():
-            with gr.Column():
-                inp_audio1 = gr.Audio(value='./test_inputs/dragon-growl-37570.wav',type='filepath',elem_id='output-audio')
-                output_img7 = gr.Image(type='filepath',elem_id='img')
-                output_text7 = gr.Textbox(label="Output Text", elem_id="output-text")
-                btn_audio1 = gr.Button("Retrieve")
-
-            with gr.Column():
-                inp_audio2 = gr.Audio(value='./test_inputs/car_audio.wav',type='filepath',elem_id='output-audio')
-                output_img8 = gr.Image(type='filepath',elem_id='img')
-                output_text8 = gr.Textbox(label="Output Text", elem_id="output-text")
-                btn_audio2 = gr.Button("Retrieve")
-            
-            with gr.Column():
-                inp_audio3 = gr.Audio(value='./test_inputs/bird_audio.wav',type='filepath',elem_id='output-audio')
-                output_img9 = gr.Image(type='filepath',elem_id='img')
-                output_text9 = gr.Textbox(label="Output Text", elem_id="output-text")
-                btn_audio3 = gr.Button("Retrieve")
-        
-    # Click actions for buttons/Textboxes
-    btn_img1.click(process_image, inputs=[inp1],outputs=[output_text1,output_audio1])
-    btn_img2.click(process_image, inputs=[inp2],outputs=[output_text2,output_audio2])
-    btn_img3.click(process_image, inputs=[inp3],outputs=[output_text3,output_audio3])
-        
-    input_txt1.submit(process_text, inputs=[input_txt1],outputs=[output_img1,output_audio4])
-
-    btn_audio1.click(process_audio, inputs=[inp_audio1],outputs=[output_img7,output_text7])
-    btn_audio2.click(process_audio, inputs=[inp_audio2],outputs=[output_img8,output_text8])
-    btn_audio3.click(process_audio, inputs=[inp_audio3],outputs=[output_img9,output_text9])
 
-if __name__ == "__main__":
-    app.launch(share=True,allowed_paths=['./test_inputs'])
+im_to_at = gr.Interface(
+    process_image,
+    gr.Image(type="filepath", value=image_paths[0]),
+    [gr.Text(label="Output Text"), gr.Audio(label="Output Audio")],
+    examples=image_paths,
+    allow_flagging="never",
+)
+txt_to_ia = gr.Interface(
+    process_text,
+    gr.Textbox(label="Enter a prompt:"),
+    [gr.Image(label="Output Image"), gr.Audio(label="Output Audio")],
+    allow_flagging="never",
+    examples=text_list,
+)
+a_to_it = gr.Interface(
+    process_audio,
+    gr.Audio(type="filepath", value=audio_paths[0]),
+    [gr.Image(label="Output Image"), gr.Text(label="Output Text")],
+    examples=audio_paths,
+    allow_flagging="never",
+)
+demo = gr.TabbedInterface(
+    [im_to_at, txt_to_ia, a_to_it],
+    ["Image to Text/Audio", "Text to Image/Audio", "Audio to Image/Text"],
+)
 
+if __name__ == "__main__":
+    demo.launch(share=True, allowed_paths=[f"{base_path}/test_inputs/"])

downloader.py

@@ -0,0 +1,66 @@
+import requests
+import os
+from pathlib import Path
+
+# URL of the raw audio file on GitHub
+audio_file_urls = [
+    "https://github.com/raghavdixit99/assets/raw/main/bird_audio.wav",
+    "https://github.com/raghavdixit99/assets/raw/main/dragon-growl-37570.wav",
+    "https://github.com/raghavdixit99/assets/raw/main/car_audio.wav",
+]
+image_urls = [
+    "https://github.com/raghavdixit99/assets/assets/34462078/abf47cc4-d979-4aaa-83be-53a2115bf318",
+    "https://github.com/raghavdixit99/assets/assets/34462078/93be928e-522b-4e37-889d-d4efd54b2112",
+    "https://github.com/raghavdixit99/assets/assets/34462078/025deaff-632a-4829-a86c-3de6e326402f",
+]
+
+base_path = os.path.dirname(os.path.abspath(__file__))
+
+
+# Local path where you want to save the .wav file
+def dowload_and_save_audio():
+    audio_pths = []
+    for url in audio_file_urls:
+        filename = url.split("/")[-1]
+        local_file_path = Path(f"{base_path}/test_inputs/{filename}")
+        local_file_path.parent.mkdir(parents=True, exist_ok=True)
+        # Perform the GET request
+        response = requests.get(url)
+
+        # Check if the request was successful
+        if response.status_code == 200:
+            # Write the content of the response to a local file
+            with open(local_file_path, "wb") as audio_file:
+                audio_file.write(response.content)
+                audio_pths.append(str(local_file_path))
+            print(
+                f"Audio file downloaded successfully and saved as '{local_file_path}'."
+            )
+        else:
+            print(f"Failed to download file. Status code: {response.status_code}")
+    return audio_pths
+
+
+def dowload_and_save_image():
+    image_paths = []
+    for url in image_urls:
+        filename = url.split("/")[-1]
+        local_file_path = Path(f"{base_path}/test_inputs/{filename}.jpeg")
+
+        local_file_path.parent.mkdir(parents=True, exist_ok=True)
+        # Perform the GET request
+        response = requests.get(url)
+
+        # Check if the request was successful
+        if response.status_code == 200:
+            # Write the content of the response to a local file
+            with open(local_file_path, "wb") as image_file:
+                image_file.write(response.content)
+                image_paths.append(str(local_file_path))
+            print(
+                f"Image file downloaded successfully and saved as '{local_file_path}'."
+            )
+        else:
+            print(f"Failed to download file. Status code: {response.status_code}")
+
+    return image_paths