app add

app.py

@@ -0,0 +1,156 @@
+import lancedb
+import lancedb.embeddings.imagebind
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+import pandas as pd
+
+model = get_registry().get("imagebind").create()
+
+class TextModel(LanceModel):
+   text: str 
+   image_uri: str = model.SourceField()
+   audio_path: str 
+   vector: Vector(model.ndims()) = model.VectorField()
+
+text_list=["A dragon", "A car", "A bird"]
+image_paths=["./test_inputs/dragon.jpg",
+              "./test_inputs/car.jpg",
+              "./test_inputs/bird_image.jpg"]
+audio_paths=["./test_inputs/dragon-growl-37570.wav", "./test_inputs/car_audio.wav",
+              "./test_inputs/bird_audio.wav"]
+
+# Load data
+inputs = [
+    {
+        "text": a,
+        "audio_path":b,
+        "image_uri":c
+    } for a,
+    b,
+    c in zip(text_list,
+                       audio_paths,
+                       image_paths)
+]
+
+
+db = lancedb.connect("~/lancedb")
+table = db.create_table("img_bind",schema=TextModel)
+table.add(inputs)
+
+import gradio as gr
+
+def process_image(inp_img) -> str:
+
+    actual = (
+    table.search(inp_img, vector_column_name="vector")
+    .limit(1)
+    .to_pydantic(TextModel)[0]
+    )
+
+    return actual.text, actual.audio_path
+
+def process_text(inp_text) -> str:
+
+    actual = (
+    table.search(inp_text, vector_column_name="vector")
+    .limit(1)
+    .to_pydantic(TextModel)[0]
+    )
+
+    return actual.image_uri, actual.audio_path
+
+def process_audio(inp_audio) -> str:
+
+    actual = (
+    table.search(inp_audio, vector_column_name="vector")
+    .limit(1)
+    .to_pydantic(TextModel)[0]
+    )
+
+    return actual.image_uri, actual.text
+
+css = """
+output-audio, output-text { 
+    display: none; 
+}
+img { 
+width: 300px; 
+height: 450px; 
+object-fit: cover; 
+
+"""
+with gr.Blocks(css=css) as app:
+    # Using Markdown for custom CSS (optional)
+    with gr.Tab("Image to Text and Audio"):
+        with gr.Row():
+            with gr.Column():
+                inp1 = gr.Image(value='./test_inputs/dragon.jpg',type='filepath',elem_id='img')
+                output_audio1 = gr.Audio(label="Output Audio", elem_id="output-audio")
+                output_text1 = gr.Textbox(label="Output Text", elem_id="output-text")
+                btn_img1 = gr.Button("Retrieve")
+                
+                # output_audio1 = gr.Audio(label="Output Audio 1", elem_id="output-audio1")
+            with gr.Column():
+                inp2 = gr.Image(value='./test_inputs/car.jpg',type='filepath',elem_id='img')
+                output_audio2 = gr.Audio(label="Output Audio", elem_id="output-audio")
+                output_text2 = gr.Textbox(label="Output Text", elem_id="output-text")
+                btn_img2 = gr.Button("Retrieve")
+            
+            with gr.Column():
+                inp3 = gr.Image(value='./test_inputs/bird_image.jpg',type='filepath',elem_id='img')
+                output_audio3 = gr.Audio(label="Output Audio", elem_id="output-audio")
+                output_text3 = gr.Textbox(label="Output Text", elem_id="output-text")
+                btn_img3 = gr.Button("Retrieve")
+    
+    with gr.Tab("Text to Image and Audio"):
+        with gr.Row():
+            with gr.Column():
+                input_txt1 = gr.Textbox(label="Enter a prompt:", elem_id="output-text")
+                output_audio4 = gr.Audio(label="Output Audio", elem_id="output-audio")
+                output_img1 = gr.Image(type='filepath',elem_id='img')
+                
+            # with gr.Column():
+            #     input_txt2 = gr.Textbox(label="Enter a prompt:", elem_id="output-text")
+            #     output_audio5 = gr.Audio(label="Output Audio", elem_id="output-audio")
+            #     output_img2 = gr.Image(type='filepath',elem_id='img')
+             
+            
+            # with gr.Column():
+            #     input_txt3 = gr.Textbox(label="Enter a prompt:", elem_id="output-text")
+            #     output_audio6 = gr.Audio(label="Output Audio", elem_id="output-audio")
+            #     output_img3 = gr.Image(type='filepath',elem_id='img')
+
+    with gr.Tab("Audio to Image and Text"):
+        with gr.Row():
+            with gr.Column():
+                inp_audio1 = gr.Audio(value='./test_inputs/dragon-growl-37570.wav',type='filepath',elem_id='output-audio')
+                output_img7 = gr.Image(type='filepath',elem_id='img')
+                output_text7 = gr.Textbox(label="Output Text", elem_id="output-text")
+                btn_audio1 = gr.Button("Retrieve")
+
+            with gr.Column():
+                inp_audio2 = gr.Audio(value='./test_inputs/car_audio.wav',type='filepath',elem_id='output-audio')
+                output_img8 = gr.Image(type='filepath',elem_id='img')
+                output_text8 = gr.Textbox(label="Output Text", elem_id="output-text")
+                btn_audio2 = gr.Button("Retrieve")
+            
+            with gr.Column():
+                inp_audio3 = gr.Audio(value='./test_inputs/bird_audio.wav',type='filepath',elem_id='output-audio')
+                output_img9 = gr.Image(type='filepath',elem_id='img')
+                output_text9 = gr.Textbox(label="Output Text", elem_id="output-text")
+                btn_audio3 = gr.Button("Retrieve")
+        
+    # Click actions for buttons/Textboxes
+    btn_img1.click(process_image, inputs=[inp1],outputs=[output_text1,output_audio1])
+    btn_img2.click(process_image, inputs=[inp2],outputs=[output_text2,output_audio2])
+    btn_img3.click(process_image, inputs=[inp3],outputs=[output_text3,output_audio3])
+        
+    input_txt1.submit(process_text, inputs=[input_txt1],outputs=[output_img1,output_audio4])
+
+    btn_audio1.click(process_audio, inputs=[inp_audio1],outputs=[output_img7,output_text7])
+    btn_audio2.click(process_audio, inputs=[inp_audio2],outputs=[output_img8,output_text8])
+    btn_audio3.click(process_audio, inputs=[inp_audio3],outputs=[output_img9,output_text9])
+
+if __name__ == "__main__":
+    app.launch(share=True)
+

requirements.txt

@@ -0,0 +1,4 @@
+lancedb
+gradio 
+pandas
+imagebind@git+https://github.com/raghavdixit99/ImageBind.git