app.py

from io import BytesIO
import string
import gradio as gr
import requests
from caption_anything import CaptionAnything
import torch
import json
import sys
import argparse
from caption_anything import parse_augment
import numpy as np
import PIL.ImageDraw as ImageDraw
from image_editing_utils import create_bubble_frame
import copy
from tools import mask_painter
from PIL import Image
import os
from captioner import build_captioner
from segment_anything import sam_model_registry
from text_refiner import build_text_refiner
from segmenter import build_segmenter


def download_checkpoint(url, folder, filename):
    os.makedirs(folder, exist_ok=True)
    filepath = os.path.join(folder, filename)

    if not os.path.exists(filepath):
        response = requests.get(url, stream=True)
        with open(filepath, "wb") as f:
            for chunk in response.iter_content(chunk_size=8192):
                if chunk:
                    f.write(chunk)

    return filepath


title = """<p><h1 align="center">Caption-Anything</h1></p>
"""
description = """<p>Gradio demo for Caption Anything, image to dense captioning generation with various language styles. To use it, simply upload your image, or click one of the examples to load them. Code: https://github.com/ttengwang/Caption-Anything <a href="https://huggingface.co/spaces/TencentARC/Caption-Anything?duplicate=true"><img style="display: inline; margin-top: 0em; margin-bottom: 0em" src="https://bit.ly/3gLdBN6" alt="Duplicate Space" /></a></p>"""

examples = [
    ["test_img/img35.webp"],
    ["test_img/img2.jpg"],
    ["test_img/img5.jpg"],
    ["test_img/img12.jpg"],
    ["test_img/img14.jpg"],
    ["test_img/img0.png"],
    ["test_img/img1.jpg"],
]

seg_model_map = {
    'base': 'vit_b',
    'large': 'vit_l',
    'huge': 'vit_h'
}
ckpt_url_map = {
    'vit_b': 'https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth',
    'vit_l': 'https://dl.fbaipublicfiles.com/segment_anything/sam_vit_l_0b3195.pth',
    'vit_h': 'https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth'
}
os.makedirs('result', exist_ok=True)
args = parse_augment()

checkpoint_url = ckpt_url_map[seg_model_map[args.segmenter]]
folder = "segmenter"
filename = os.path.basename(checkpoint_url)
args.segmenter_checkpoint = os.path.join(folder, filename)

download_checkpoint(checkpoint_url, folder, filename)

# args.device = 'cuda:5'
# args.disable_gpt = True
# args.enable_reduce_tokens = False
# args.port=20322
# args.captioner = 'blip'
# args.regular_box = True
shared_captioner = build_captioner(args.captioner, args.device, args)
shared_sam_model = sam_model_registry[seg_model_map[args.segmenter]](checkpoint=args.segmenter_checkpoint).to(args.device)


def build_caption_anything_with_models(args, api_key="", captioner=None, sam_model=None, text_refiner=None, session_id=None):
    segmenter = build_segmenter(args.segmenter, args.device, args, model=sam_model)
    captioner = captioner
    if session_id is not None:
        print('Init caption anything for session {}'.format(session_id))
    return CaptionAnything(args, api_key, captioner=captioner, segmenter=segmenter, text_refiner=text_refiner)


def init_openai_api_key(api_key=""):
    text_refiner = None
    if api_key and len(api_key) > 30:
        try:
            text_refiner = build_text_refiner(args.text_refiner, args.device, args, api_key)
            text_refiner.llm('hi') # test
        except:
            text_refiner = None
    openai_available = text_refiner is not None
    return gr.update(visible = openai_available), gr.update(visible = openai_available), gr.update(visible = openai_available), gr.update(visible = True), gr.update(visible = True), gr.update(visible = True), text_refiner


def get_prompt(chat_input, click_state, click_mode):
    inputs = json.loads(chat_input)
    if click_mode == 'Continuous':
        points = click_state[0]
        labels = click_state[1]
        for input in inputs:
            points.append(input[:2])
            labels.append(input[2])
    elif click_mode == 'Single':
        points = []
        labels = []
        for input in inputs:
            points.append(input[:2])
            labels.append(input[2])
        click_state[0] = points
        click_state[1] = labels
    else:
        raise NotImplementedError

    prompt = {
        "prompt_type":["click"],
        "input_point":click_state[0],
        "input_label":click_state[1],
        "multimask_output":"True",
    }
    return prompt

def update_click_state(click_state, caption, click_mode):
    if click_mode == 'Continuous':
        click_state[2].append(caption)
    elif click_mode == 'Single':
        click_state[2] = [caption]
    else:
        raise NotImplementedError


def chat_with_points(chat_input, click_state, chat_state, state, text_refiner, img_caption):
    if text_refiner is None:
        response = "Text refiner is not initilzed, please input openai api key."
        state = state + [(chat_input, response)]
        return state, state, chat_state

    points, labels, captions = click_state
    # point_chat_prompt = "I want you act as a chat bot in terms of image. I will give you some points (w, h) in the image and tell you what happed on the point in natural language. Note that (0, 0) refers to the top-left corner of the image, w refers to the width and h refers the height. You should chat with me based on the fact in the image instead of imagination. Now I tell you the points with their visual description:\n{points_with_caps}\nNow begin chatting!"
    suffix = '\nHuman: {chat_input}\nAI: '
    qa_template = '\nHuman: {q}\nAI: {a}'
    # # "The image is of width {width} and height {height}." 
    point_chat_prompt = "I am an AI trained to chat with you about an image. I am greate at what is going on in any image based on the image information your provide. The overall image description is \"{img_caption}\". You will also provide me objects in the image in details, i.e., their location and visual descriptions. Here are the locations and descriptions of events that happen in the image: {points_with_caps} \n Now, let's chat!"
    prev_visual_context = ""
    pos_points = []
    pos_captions = []
    for i in range(len(points)):
        if labels[i] == 1:
            pos_points.append(f"({points[i][0]}, {points[i][0]})")
            pos_captions.append(captions[i])
            prev_visual_context = prev_visual_context + '\n' + 'There is an event described as  \"{}\" locating at {}'.format(pos_captions[-1], ', '.join(pos_points))

    context_length_thres = 500
    prev_history = ""
    for i in range(len(chat_state)):
        q, a = chat_state[i]
        if len(prev_history) < context_length_thres:
            prev_history = prev_history + qa_template.format(**{"q": q, "a": a})
        else:
            break
    chat_prompt = point_chat_prompt.format(**{"img_caption":img_caption,"points_with_caps": prev_visual_context}) + prev_history + suffix.format(**{"chat_input": chat_input})
    print('\nchat_prompt: ', chat_prompt)
    response = text_refiner.llm(chat_prompt)
    state = state + [(chat_input, response)]
    chat_state = chat_state + [(chat_input, response)]
    return state, state, chat_state

def inference_seg_cap(image_input, point_prompt, click_mode, enable_wiki, language, sentiment, factuality,
                      length, image_embedding, state, click_state, original_size, input_size, text_refiner, evt:gr.SelectData):

    model = build_caption_anything_with_models(
        args,
        api_key="",
        captioner=shared_captioner,
        sam_model=shared_sam_model,
        text_refiner=text_refiner,
        session_id=iface.app_id
    )

    model.segmenter.image_embedding = image_embedding
    model.segmenter.predictor.original_size = original_size
    model.segmenter.predictor.input_size = input_size
    model.segmenter.predictor.is_image_set = True

    if point_prompt == 'Positive':
        coordinate = "[[{}, {}, 1]]".format(str(evt.index[0]), str(evt.index[1]))
    else:
        coordinate = "[[{}, {}, 0]]".format(str(evt.index[0]), str(evt.index[1]))

    controls = {'length': length,
                'sentiment': sentiment,
                'factuality': factuality,
                'language': language}

    # click_coordinate = "[[{}, {}, 1]]".format(str(evt.index[0]), str(evt.index[1])) 
    # chat_input = click_coordinate
    prompt = get_prompt(coordinate, click_state, click_mode)
    print('prompt: ', prompt, 'controls: ', controls)

    enable_wiki = True if enable_wiki in ['True', 'TRUE', 'true', True, 'Yes', 'YES', 'yes'] else False
    out = model.inference(image_input, prompt, controls, disable_gpt=True, enable_wiki=enable_wiki)
    state = state + [("Image point: {}, Input label: {}".format(prompt["input_point"], prompt["input_label"]), None)]
    # for k, v in out['generated_captions'].items():
    #     state = state + [(f'{k}: {v}', None)]
    state = state + [(None, "raw_caption: {}".format(out['generated_captions']['raw_caption']))]
    wiki = out['generated_captions'].get('wiki', "")

    update_click_state(click_state, out['generated_captions']['raw_caption'], click_mode)
    text = out['generated_captions']['raw_caption']
    # draw = ImageDraw.Draw(image_input)
    # draw.text((evt.index[0], evt.index[1]), text, textcolor=(0,0,255), text_size=120)
    input_mask = np.array(out['mask'].convert('P'))
    image_input = mask_painter(np.array(image_input), input_mask)
    origin_image_input = image_input
    image_input = create_bubble_frame(image_input, text, (evt.index[0], evt.index[1]))

    yield state, state, click_state, chat_input, image_input, wiki
    if not args.disable_gpt and model.text_refiner:
        refined_caption = model.text_refiner.inference(query=text, controls=controls, context=out['context_captions'], enable_wiki=enable_wiki)
        # new_cap = 'Original: ' + text + '. Refined: ' + refined_caption['caption']
        new_cap = refined_caption['caption']
        wiki = refined_caption['wiki']
        state = state + [(None, f"caption: {new_cap}")]
        refined_image_input = create_bubble_frame(origin_image_input, new_cap, (evt.index[0], evt.index[1]))
        yield state, state, click_state, chat_input, refined_image_input, wiki


def upload_callback(image_input, state):    
    chat_state = []
    click_state = [[], [], []]
    res = 1024
    width, height = image_input.size
    ratio = min(1.0 * res / max(width, height), 1.0)
    if ratio < 1.0:
        image_input = image_input.resize((int(width * ratio), int(height * ratio)))
        print('Scaling input image to {}'.format(image_input.size))
    state = [] + [(None, 'Image size: ' + str(image_input.size))]
    model = build_caption_anything_with_models(
        args,
        api_key="",
        captioner=shared_captioner,
        sam_model=shared_sam_model,
        session_id=iface.app_id
    )
    model.segmenter.set_image(image_input)
    image_embedding = model.segmenter.image_embedding
    original_size = model.segmenter.predictor.original_size
    input_size = model.segmenter.predictor.input_size
    img_caption, _ = model.captioner.inference_seg(image_input)
    return state, state, chat_state, image_input, click_state, image_input, image_embedding, original_size, input_size, img_caption

with gr.Blocks(
        css='''
    #image_upload{min-height:400px}
    #image_upload [data-testid="image"], #image_upload [data-testid="image"] > div{min-height: 600px}
    '''
) as iface:
    state = gr.State([])
    click_state = gr.State([[],[],[]])
    chat_state = gr.State([])
    origin_image = gr.State(None)
    image_embedding = gr.State(None)
    text_refiner = gr.State(None)
    original_size = gr.State(None)
    input_size = gr.State(None)
    img_caption = gr.State(None)

    gr.Markdown(title)
    gr.Markdown(description)

    with gr.Row():
        with gr.Column(scale=1.0):
            with gr.Column(visible=False) as modules_not_need_gpt:
                image_input = gr.Image(type="pil", interactive=True, elem_id="image_upload")
                example_image = gr.Image(type="pil", interactive=False, visible=False)
                with gr.Row(scale=1.0):
                    with gr.Row(scale=0.4):
                        point_prompt = gr.Radio(
                            choices=["Positive",  "Negative"],
                            value="Positive",
                            label="Point Prompt",
                            interactive=True)
                        click_mode = gr.Radio(
                            choices=["Continuous",  "Single"],
                            value="Continuous",
                            label="Clicking Mode",
                            interactive=True)
                    with gr.Row(scale=0.4):
                        clear_button_clike = gr.Button(value="Clear Clicks", interactive=True)
                        clear_button_image = gr.Button(value="Clear Image", interactive=True)
            with gr.Column(visible=False) as modules_need_gpt:
                with gr.Row(scale=1.0):
                    language = gr.Dropdown(['English', 'Chinese', 'French', "Spanish", "Arabic", "Portuguese", "Cantonese"], value="English", label="Language", interactive=True)
                    sentiment = gr.Radio(
                        choices=["Positive", "Natural", "Negative"],
                        value="Natural",
                        label="Sentiment",
                        interactive=True,
                    )
                with gr.Row(scale=1.0):
                    factuality = gr.Radio(
                        choices=["Factual", "Imagination"],
                        value="Factual",
                        label="Factuality",
                        interactive=True,
                    )
                    length = gr.Slider(
                        minimum=10,
                        maximum=80,
                        value=10,
                        step=1,
                        interactive=True,
                        label="Generated Caption Length",
                    )
                    enable_wiki = gr.Radio(
                        choices=["Yes",  "No"],
                        value="No",
                        label="Enable Wiki",
                        interactive=True)
            with gr.Column(visible=True) as modules_not_need_gpt3:
                gr.Examples(
                    examples=examples,
                    inputs=[example_image],
                )
        with gr.Column(scale=0.5):
            openai_api_key = gr.Textbox(
                placeholder="Input openAI API key",
                show_label=False,
                label = "OpenAI API Key",
                lines=1,
                type="password")
            with gr.Row(scale=0.5):
                enable_chatGPT_button = gr.Button(value="Run with ChatGPT", interactive=True, variant='primary')
                disable_chatGPT_button = gr.Button(value="Run without ChatGPT (Faster)", interactive=True, variant='primary')
            with gr.Column(visible=False) as modules_need_gpt2:
                wiki_output = gr.Textbox(lines=5, label="Wiki", max_lines=5)
            with gr.Column(visible=False) as modules_not_need_gpt2:
                chatbot = gr.Chatbot(label="Chat about Selected Object",).style(height=550,scale=0.5)
                with gr.Column(visible=False) as modules_need_gpt3:
                    chat_input = gr.Textbox(show_label=False, placeholder="Enter text and press Enter").style(container=False)
                    with gr.Row():
                        clear_button_text = gr.Button(value="Clear Text", interactive=True)
                        submit_button_text = gr.Button(value="Submit", interactive=True, variant="primary")

    openai_api_key.submit(init_openai_api_key, inputs=[openai_api_key], outputs=[modules_need_gpt,modules_need_gpt2, modules_need_gpt3, modules_not_need_gpt, modules_not_need_gpt2, modules_not_need_gpt3, text_refiner])
    enable_chatGPT_button.click(init_openai_api_key, inputs=[openai_api_key], outputs=[modules_need_gpt,modules_need_gpt2, modules_need_gpt3, modules_not_need_gpt, modules_not_need_gpt2, modules_not_need_gpt3, text_refiner])
    disable_chatGPT_button.click(init_openai_api_key, outputs=[modules_need_gpt,modules_need_gpt2, modules_need_gpt3, modules_not_need_gpt, modules_not_need_gpt2, modules_not_need_gpt3, text_refiner])

    clear_button_clike.click(
        lambda x: ([[], [], []], x, ""),
        [origin_image],
        [click_state, image_input, wiki_output],
        queue=False,
        show_progress=False
    )
    clear_button_image.click(
        lambda: (None, [], [], [], [[], [], []], "", "", ""),
        [],
        [image_input, chatbot, state, chat_state, click_state, wiki_output, origin_image, img_caption],
        queue=False,
        show_progress=False
    )
    clear_button_text.click(
        lambda: ([], [], [[], [], [], []], []),
        [],
        [chatbot, state, click_state, chat_state],
        queue=False,
        show_progress=False
    )
    image_input.clear(
        lambda: (None, [], [], [], [[], [], []], "", "", ""),
        [],
        [image_input, chatbot, state, chat_state, click_state, wiki_output, origin_image, img_caption],
        queue=False,
        show_progress=False
    )

    image_input.upload(upload_callback,[image_input, state], [chatbot, state, chat_state, origin_image, click_state, image_input, image_embedding, original_size, input_size, img_caption])
    chat_input.submit(chat_with_points, [chat_input, click_state, chat_state, state, text_refiner, img_caption], [chatbot, state, chat_state])
    chat_input.submit(lambda: "", None, chat_input)
    example_image.change(upload_callback,[example_image, state], [chatbot, state, chat_state, origin_image, click_state, image_input, image_embedding, original_size, input_size, img_caption])

    # select coordinate
    image_input.select(inference_seg_cap,
                       inputs=[
                           origin_image,
                           point_prompt,
                           click_mode,
                           enable_wiki,
                           language,
                           sentiment,
                           factuality,
                           length,
                           image_embedding,
                           state,
                           click_state,
                           original_size,
                           input_size,
                           text_refiner
                       ],
                       outputs=[chatbot, state, click_state, chat_input, image_input, wiki_output],
                       show_progress=False, queue=True)

iface.queue(concurrency_count=5, api_open=False, max_size=10)
iface.launch(server_name="0.0.0.0", enable_queue=True)