adding app file

app.py

@@ -0,0 +1,112 @@
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch.nn.functional as F
+from PIL import Image
+import uform
+
+model = uform.get_model('unum-cloud/uform-vl-english')
+
+def find_score(img, txt, if_fine_grained):
+    txt = model.preprocess_text(txt)
+    img = model.preprocess_image(img)
+    txt_features, txt_emb = model.encode_text(txt, return_features = True)
+    img_features, img_emb = model.encode_image(img, return_features = True)
+    if if_fine_grained:
+        joint_embedding = model.encode_multimodal(
+        image_features=img_features,
+        text_features=txt_features,
+        attention_mask=txt['attention_mask'])
+        score = model.get_matching_scores(joint_embedding)
+    else:
+        score = F.cosine_similarity(txt_emb, img_emb)
+    return score
+
+def find_score_img(img1, img2, if_fine_grained):
+    img1 = model.preprocess_image(img1)
+    img2 = model.preprocess_image(img2)
+    img_features1, img_emb1 = model.encode_image(img1, return_features = True)
+    img_features2, img_emb2 = model.encode_image(img2, return_features = True)
+    if if_fine_grained:
+        joint_embedding = model.encode_multimodal(
+        image_features=img_features1,
+        text_features=img_features2,
+        attention_mask=img1['attention_mask'])
+        score = model.get_matching_scores(joint_embedding)
+    else:
+        score = F.cosine_similarity(img_emb1, img_emb2)
+    return score
+
+def find_score_txt(txt1, txt2, if_fine_grained):
+    txt1 = model.preprocess_text(txt1)
+    txt2 = model.preprocess_text(txt2)
+    txt_features1, txt_emb1 = model.encode_text(txt1, return_features = True)
+    txt_features2, txt_emb2 = model.encode_text(txt2, return_features = True)
+    if if_fine_grained:
+        joint_embedding = model.encode_multimodal(
+        image_features=txt_features1,
+        text_features=txt_features2,
+        attention_mask=txt1['attention_mask'])
+        score = model.get_matching_scores(joint_embedding)
+    else:
+        score = F.cosine_similarity(txt_emb1, txt_emb2)
+    return score
+    
+with gr.Blocks(theme = gr.themes.Glass()) as demo_mix:
+    gr.Markdown('# Find similarity between images and text.')
+    with gr.Row():
+        with gr.Column():
+            img_input = gr.Image(source = 'upload', type = 'pil', label = "Drop your image here", shape = [256, 256])
+        with gr.Column():
+            txt_input = gr.Textbox(label = 'Enter your text here:' , lines = 1,)
+            if_fine_grained = gr.Checkbox(label = "Check for a more fine-grained comparison")
+            btn = gr.Button("Find similarity")
+    score = gr.Number(label='Similarity score')
+    btn.click(find_score, inputs=[img_input, txt_input, if_fine_grained], outputs=[score])
+    gr.Markdown('If the box for a more fine-grained comparison is checked, the similarity score will be in (0,1), otherwise, it will be in (-1,1)')
+    gr.Markdown('### Image examples')
+    gr.Examples(['imgs/red_panda.jpg', 'imgs/trash_raccoon.jpg', 'imgs/baby_panda.jpg', 'imgs/rocket.jpg'], inputs=[img_input])
+    gr.Markdown('### Text examples')
+    gr.Examples(['baby red panda staring into the camera', \
+        'trash raccoon peeking from a trash bin', 
+        'a cartoonish raccoon wearing a blue and red suit', 
+        "a person holding a baby panda"], inputs=[txt_input])  
+
+
+with gr.Blocks() as demo_img:
+    gr.Markdown('# Find similarity between images.')
+    with gr.Row():
+        with gr.Column():
+            img_input1 = gr.Image(source = 'upload', type = 'pil', label = "Drop your image here", shape = [256, 256])
+            if_fine_grained = gr.Checkbox(label = "Check for a more fine-grained comparison")
+        with gr.Column():
+            img_input2 = gr.Image(source = 'upload', type = 'pil', label = "Drop your image here", shape = [256, 256])
+            btn = gr.Button("Find similarity")
+            score = gr.Number(label='Similarity score')
+    btn.click(find_score_img, inputs=[img_input1, img_input2, if_fine_grained], outputs=[score])
+    gr.Markdown('If the box for a more fine-grained comparison is checked, the similarity score will be in (0,1), otherwise, it will be in (-1,1)')
+    gr.Markdown('### Image examples')
+    gr.Examples(['imgs/red_panda.jpg', 'imgs/trash_raccoon.jpg', 'imgs/baby_panda.jpg', 'imgs/rocket.jpg'], inputs=[img_input1])
+
+
+with gr.Blocks() as demo_txt:
+    gr.Markdown('# Find similarity between short descriptions.')
+    with gr.Row():
+        with gr.Column():
+            txt_input1 = gr.Textbox(label = 'Enter your text here:' , lines = 1)
+            txt_input2 = gr.Textbox(label = 'Enter your text here:' , lines = 1)
+        with gr.Column():
+            if_fine_grained = gr.Checkbox(label = "Check for a more fine-grained comparison")
+            btn = gr.Button("Find similarity")
+            score = gr.Number(label='Similarity score')
+    btn.click(find_score_txt, inputs=[img_input, txt_input, if_fine_grained], outputs=[score])
+    gr.Markdown('If the box for a more fine-grained comparison is checked, the similarity score will be in (0,1), otherwise, it will be in (-1,1)')
+    gr.Markdown('### Text examples')
+    gr.Examples(['baby red panda staring into the camera', \
+        'trash raccoon peeking from a trash bin', 
+        'a cartoonish raccoon wearing a blue and red suit', 
+        "a person holding a baby panda"], inputs=[txt_input1]) 
+
+demo = gr.TabbedInterface([demo_mix, demo_img, demo_txt], ["img2txt", "img2img", "txt2txt"])
+
+if __name__ == "__main__":
+    demo.launch(share=True)