app.py

import os
import app_configs as configs
from feedback import Feedback
import service
import gradio as gr
import numpy as np
import cv2 
from PIL import Image
import logging 
from huggingface_hub import hf_hub_download
import torch

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()

sam = None #service.get_sam(configs.model_type, configs.model_ckpt_path, configs.device)
red = (255,0,0)
blue = (0,0,255)

def load_sam_instance():
    global sam
    if sam is None:
        gr.Info('Initialising SAM, hang in there...')
        if not os.path.exists(configs.model_ckpt_path):
            chkpt_path = hf_hub_download("ybelkada/segment-anything", configs.model_ckpt_path)
        else:
            chkpt_path = configs.model_ckpt_path
        device = configs.device
        if device is None:
            device = 'cuda' if torch.cuda.is_available() else 'cpu'
        sam = service.get_sam(configs.model_type, chkpt_path, device)
    return sam

block = gr.Blocks()
with block:
    # states
    def point_coords_empty():
        return []
    def point_labels_empty():
        return []
    raw_image = gr.Image(type='pil', visible=False)
    point_coords = gr.State(point_coords_empty)
    point_labels = gr.State(point_labels_empty)
    masks = gr.State()
    cutout_idx = gr.State(set())
    feedback = gr.State(lambda : Feedback())

    # UI
    with gr.Column():
        with gr.Row():
            input_image = gr.Image(label='Input', height=512, type='pil')
            masks_annotated_image = gr.AnnotatedImage(label='Segments', height=512)
            cutout_galary = gr.Gallery(label='Cutouts', object_fit='contain', height=512)
        with gr.Row():
            with gr.Column(scale=1):
                point_label_radio = gr.Radio(label='Point Label', choices=[1,0], value=1)
                reset_btn = gr.Button('Reset')
                run_btn = gr.Button('Run', variant = 'primary')
            with gr.Column(scale=2):
                with gr.Accordion('Provide Feedback'):
                    feedback_textbox = gr.Textbox(lines=3, show_label=False, info="Comments (Leave blank to vote without any comments)")
                    with gr.Row():
                        upvote_button = gr.Button('Upvote')
                        downvote_button = gr.Button('Downvote')
    # components
    components = {
        point_coords, point_labels, raw_image, masks, cutout_idx,
        feedback, upvote_button, downvote_button, feedback_textbox,
        input_image, point_label_radio, reset_btn, run_btn, masks_annotated_image}
    # event - init coords
    def on_reset_btn_click(raw_image):
        return raw_image, point_coords_empty(), point_labels_empty(), None, []
    reset_btn.click(on_reset_btn_click, [raw_image], [input_image, point_coords, point_labels], queue=False)

    def on_input_image_upload(input_image):
        return input_image, point_coords_empty(), point_labels_empty(), None
    input_image.upload(on_input_image_upload, [input_image], [raw_image, point_coords, point_labels], queue=False)

    # event - set coords
    def on_input_image_select(input_image, point_coords, point_labels, point_label_radio, evt: gr.SelectData):
        x, y = evt.index
        color = red if point_label_radio == 0 else blue
        img = np.array(input_image)
        cv2.circle(img, (x, y), 5, color, -1)
        img = Image.fromarray(img)
        point_coords.append([x,y])
        point_labels.append(point_label_radio)
        return img, point_coords, point_labels
    input_image.select(on_input_image_select, [input_image, point_coords, point_labels, point_label_radio], [input_image, point_coords, point_labels], queue=False)

    # event - inference
    def on_run_btn_click(inputs):
        sam = load_sam_instance()
        image = inputs[raw_image]
        if len(inputs[point_coords]) == 0:
            if configs.enable_segment_all:
                generated_masks, _ = service.predict_all(sam, image)
            else:
                raise gr.Error('Segment-all disabled, set point label(s) before running')
        else:
            generated_masks, _ = service.predict_conditioned(sam,
                                                            image, 
                                                            point_coords=np.array(inputs[point_coords]), 
                                                            point_labels=np.array(inputs[point_labels]))
        annotated = (image, [(generated_masks[i], f'Mask {i}') for i in range(len(generated_masks))])
        inputs[feedback].save_inference(
            pt_coords=inputs[point_coords], 
            pt_labels=inputs[point_labels],
            image=inputs[raw_image],
            mask=generated_masks,
            )
        return {
            masks_annotated_image:annotated, 
            masks: generated_masks, 
            cutout_idx: set(),
            feedback: inputs[feedback],
            }
    run_btn.click(on_run_btn_click, components, [masks_annotated_image, masks, cutout_idx, feedback], queue=True)

    # event - get cutout 
    def on_masks_annotated_image_select(inputs, evt:gr.SelectData):
        inputs[cutout_idx].add(evt.index)
        cutouts = [service.cutout(inputs[raw_image], inputs[masks][idx]) for idx in list(inputs[cutout_idx])]
        tight_cutouts = [service.crop_empty(cutout) for cutout in cutouts]
        inputs[feedback].save_feedback(cutout_idx=evt.index)
        return inputs[cutout_idx], tight_cutouts, inputs[feedback]
    masks_annotated_image.select(on_masks_annotated_image_select, components, [cutout_idx, cutout_galary, feedback], queue=False)

    # event - feedback
    def on_upvote_button_click(inputs):
        inputs[feedback].save_feedback(like=1, feedback_str=inputs[feedback_textbox])
        gr.Info('Thanks for your feedback')
        return {feedback:inputs[feedback],feedback_textbox:None}
    upvote_button.click(on_upvote_button_click,components,[feedback, feedback_textbox], queue=False)
    def on_downvote_button_click(inputs):
        inputs[feedback].save_feedback(like=-1, feedback_str=inputs[feedback_textbox])
        gr.Info('Thanks for your feedback')
        return {feedback:inputs[feedback],feedback_textbox:None}
    downvote_button.click(on_downvote_button_click,components,[feedback, feedback_textbox], queue=False)
if __name__ == '__main__':
    block.queue()
    block.launch()