added more applications

app.py

@@ -1,16 +1,34 @@
 import gradio as gr
 
-from tasks.asr import (
+from apps.asr import (
     mic_transcribe_interface,
     file_transcribe_interface
 )
+from apps.object_detection import obj_detection_interface
+from apps.image_captioning import img_captioning_interface
+from apps.multimodal_visual_qa import multimodal_visual_qa_interface
+from apps.ner import ner_interface
 
 app = gr.Blocks()
 
 with app:
     gr.TabbedInterface(
-        [mic_transcribe_interface, file_transcribe_interface],
-        ["Transcribe from Microphone", "Transcribe from Audio File"]
+        [
+            mic_transcribe_interface,
+            file_transcribe_interface, 
+            obj_detection_interface,
+            img_captioning_interface,
+            multimodal_visual_qa_interface,
+            ner_interface
+        ],
+        [
+            "Transcribe from Microphone",
+            "Transcribe from Audio File", 
+            "Detect Objects from an Image",
+            "Generate a Caption for an Image",
+            "Perform QA on an Image",
+            "Named Entity Recogntion"
+        ]
     )
 
 app.launch(share=True)

apps/image_captioning.py

@@ -0,0 +1,21 @@
+import gradio as gr 
+from transformers import AutoProcessor
+from transformers import BlipForConditionalGeneration 
+
+model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
+processor = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
+
+def caption_image(image):
+    inputs = processor(image, return_tensors="pt")
+    output = model.generate(**inputs)
+    caption = processor.decode(output[0], skip_special_tokens=True)
+
+    return caption 
+
+img_captioning_interface = gr.Interface(
+    fn=caption_image,
+    inputs=gr.Image(label="Input Image", type="pil"),
+    outputs=gr.Textbox(label="Predicted Caption"),
+    title="Image Caption Generator App",
+    description="This app generates a caption for an image."
+)

apps/multimodal_visual_qa.py

@@ -0,0 +1,24 @@
+import gradio as gr
+from transformers import AutoProcessor
+from transformers import BlipForQuestionAnswering
+
+model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
+processor = AutoProcessor.from_pretrained("Salesforce/blip-vqa-base")
+
+def process_image(image, question: str):
+    inputs = processor(image, question, return_tensors="pt")
+    output = model.generate(**inputs)
+    answer = processor.decode(output[0], skip_special_tokens=True)
+
+    return answer
+
+multimodal_visual_qa_interface = gr.Interface(
+    fn=process_image,
+    inputs=[
+        gr.Image(label="Input Image", type="pil"),
+        gr.Textbox(label="Enter question to prompt the image")
+    ],
+    outputs=gr.Textbox(label="Answer"),
+    title="Multimodal Visual QA Application",
+    description="This app can help you ask questions about an image"
+)

apps/ner.py

@@ -0,0 +1,19 @@
+import gradio as gr
+from transformers import pipeline
+
+ner_pipeline = pipeline("ner")
+
+examples = [
+    "Does Chicago have any stores and does Joe live here?",
+]
+
+def ner(text):
+    output = ner_pipeline(text)
+    return {"text": text, "entities": output}    
+
+ner_interface = gr.Interface(
+    ner,
+    gr.Textbox(placeholder="Enter sentence"), 
+    gr.HighlightedText(),
+    examples=examples
+)

apps/object_detection.py

@@ -0,0 +1,109 @@
+import io 
+
+import requests 
+import numpy as np
+import gradio as gr 
+from PIL import Image
+import matplotlib.pyplot as plt 
+from transformers import pipeline 
+
+# Load the pipeline
+obj_detector = pipeline(
+    task="object-detection",
+    model="facebook/detr-resnet-50"
+)
+
+# Object detection utilities
+def load_image_from_url(url: str):
+    return Image.open(requests.get(url, stream=True).raw).convert("RGB")
+
+def render_results_in_image(img, detection_results):
+    plt.figure(figsize=(16, 10))
+    plt.imshow(img)
+
+    ax = plt.gca()
+
+    for prediction in detection_results:
+        x, y = prediction["box"]["xmin"], prediction["box"]["ymin"]
+        w = prediction["box"]["xmax"] - prediction["box"]["xmin"]
+        h = prediction["box"]["ymax"] - prediction["box"]["ymin"]
+
+        ax.add_patch(
+            plt.Rectangle(
+                (x, y),
+                w,
+                h,
+                fill=False,
+                color="green",
+                linewidth=2
+            )
+        )
+
+        ax.text(
+            x,
+            y,
+            f"{prediction['label']}: {round(prediction['score']*100, 1)}%"
+        )
+    plt.axis("off")
+
+    # save the modified image to a BytesIO object
+    img_buf = io.BytesIO()
+    plt.savefig(img_buf, format="png",
+                bbox_inches="tight",
+                pad_inches=0)
+    img_buf.seek(0)
+    modified_image = Image.open(img_buf)
+
+    # close the plot to prevent it from being displayed
+    plt.close()
+
+    return modified_image 
+
+def summarize_detection_results(detection_results):
+    summary = {}
+
+    for prediction in detection_results:
+        label = prediction["label"]
+
+        if label in summary:
+            summary[label] += 1
+        else:
+            summary[label] = 1 
+
+    summary_string = "In this image, there are "
+    
+    for i, (label, count) in enumerate(summary.items()):
+        summary_string += f"{str(count)} {label}"
+
+        if count > 1:
+            summary_string += "s"
+        
+        summary_string += ", "
+
+        if i == len(summary) - 2:
+            summary_string += "and "
+
+    # remove the trailing comma and space
+    summary_string = summary_string.rstrip(", ") + "."
+
+    return summary_string 
+
+def detect_objects(image):
+    detection_results = obj_detector(image)
+
+    processed_image = render_results_in_image(image, detection_results)
+
+    summary_string = summarize_detection_results(detection_results)
+
+    return processed_image, summary_string 
+
+obj_detection_interface = gr.Interface(
+    fn=detect_objects,
+    inputs=gr.Image(label="Input Image", type="pil"),
+    outputs=[
+        gr.Image(label="Output image with predicted objects", type="pil"),
+        gr.Textbox(label="Object detection summary")
+    ],
+    title="Object Detection Application",
+    description="This app detects objects from an image."
+)