Add NER highlighting

app.py

@@ -48,6 +48,36 @@ def run_demo(image, text):
     for i, token in enumerate(text_tokens_decoded):
         highlighted_text.append((str(token), float(text_scores[i])))
 
+    return overlapped, highlighted_text
+
+
+# Default demo:
+input_img = gr.inputs.Image(type='pil', label="Original Image")
+input_txt = "text"
+inputs = [input_img, input_txt]
+
+outputs = [gr.inputs.Image(type='pil', label="Output Image"), "highlight"]
+
+
+iface = gr.Interface(fn=run_demo,
+                     inputs=inputs,
+                     outputs=outputs,
+                     title="CLIP Grounding Explainability",
+                     description="A demonstration based on the Generic Attention-model Explainability method for Interpreting Bi-Modal Transformers by Chefer et al. (2021): https://github.com/hila-chefer/Transformer-MM-Explainability.",
+                     examples=[["example_images/London.png", "London Eye"],
+                               ["example_images/London.png", "Big Ben"],
+                               ["example_images/harrypotter.png", "Harry"],
+                               ["example_images/harrypotter.png", "Hermione"],
+                               ["example_images/harrypotter.png", "Ron"],
+                               ["example_images/Amsterdam.png", "Amsterdam canal"],
+                               ["example_images/Amsterdam.png", "Old buildings"],
+                               ["example_images/Amsterdam.png", "Pink flowers"],
+                               ["example_images/dogs_on_bed.png", "Two dogs"],
+                               ["example_images/dogs_on_bed.png", "Book"],
+                               ["example_images/dogs_on_bed.png", "Cat"]])
+
+# NER demo:
+def NER_demo(image, text):
     # Apply NER to extract named entities, and run the explainability method
     # for each named entity.
     highlighed_entities = []
@@ -58,16 +88,16 @@ def run_demo(image, text):
 
     print(highlighed_entities)
 
-    return overlapped, highlighted_text
+    return highlighed_entities
 
 input_img = gr.inputs.Image(type='pil', label="Original Image")
 input_txt = "text"
 inputs = [input_img, input_txt]
 
-outputs = [gr.inputs.Image(type='pil', label="Output Image"), "highlight"]
+outputs = ["highlight"]
 
 
-iface = gr.Interface(fn=run_demo,
+iface_NER = gr.Interface(fn=NER_demo,
                      inputs=inputs,
                      outputs=outputs,
                      title="CLIP Grounding Explainability",
@@ -84,5 +114,6 @@ iface = gr.Interface(fn=run_demo,
                                ["example_images/dogs_on_bed.png", "Book"],
                                ["example_images/dogs_on_bed.png", "Cat"]])
 
-demo_tabs = gr.TabbedInterface([iface, iface], ["Default", "NER"])
+
+demo_tabs = gr.TabbedInterface([iface, iface_NER], ["Default", "NER"])
 demo_tabs.launch(debug=True)