initial commit

app.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+
+# ===================================================
+#
+#    Author        : Fan Zhang
+#    Email         : zhangfan@baai.ac.cn
+#    Institute     : Beijing Academy of Artificial Intelligence (BAAI)
+#    Create On     : 2023-12-11 15:34
+#    Last Modified : 2023-12-20 03:59
+#    File Name     : frontend.py
+#    Description   :
+#
+# ===================================================
+
+import argparse
+import os
+
+import gradio as gr
+from demo.generation_frontend import build_generation
+from demo.chat_frontend import build_chat
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--title", type=str, default='Emu')
+
+parser.add_argument("--host", type=str, default="0.0.0.0")
+parser.add_argument("--port", type=int, default=9002)
+parser.add_argument("--share", action="store_true")
+parser.add_argument("--controller-url", type=str, default="http://218.91.113.230:9002")
+parser.add_argument("--concurrency-count", type=int, default=8)
+parser.add_argument("--disable-chat", action="store_true")
+parser.add_argument("--disable-generate", action="store_true")
+
+args = parser.parse_args()
+
+
+if __name__ == "__main__":
+    title = "EmuV2: An Open Multimodal Generalist"
+    os.makedirs("log", exist_ok=True)
+
+    interface_list, tab_names = [], []
+    if not args.disable_generate:
+        demo_generation = build_generation(args)
+        interface_list.append(demo_generation)
+        tab_names.append("Multi-modal Generation")
+
+    if not args.disable_chat:
+        demo_chat = build_chat(args)
+        interface_list.append(demo_chat)
+        tab_names.append("Multi-modal Chat")
+
+    demo_all = gr.TabbedInterface(
+        interface_list=interface_list,
+        tab_names=tab_names,
+        title=title,
+        theme=gr.themes.Default(primary_hue="blue", secondary_hue="blue"),
+    )
+
+    demo_all.queue(
+        concurrency_count=args.concurrency_count,
+        status_update_rate=3,
+        api_open=False,
+    ).launch(
+        enable_queue=True,
+        server_name=args.host, server_port=args.port,
+        share=args.share,
+    )

demo/chat_frontend.py

@@ -0,0 +1,249 @@
+# -*- coding: utf-8 -*-
+
+# ===================================================
+#
+#    Author        : Fan Zhang
+#    Email         : zhangfan@baai.ac.cn
+#    Institute     : Beijing Academy of Artificial Intelligence (BAAI)
+#    Create On     : 2023-12-12 18:05
+#    Last Modified : 2023-12-19 15:00
+#    File Name     : chat_frontend.py
+#    Description   :
+#
+# ===================================================
+
+import json
+import io
+import time
+from PIL import Image
+import requests
+
+import gradio as gr
+
+from .meta import ConvMeta, Role, DataMeta
+from .utils import extract_frames
+from .utils import frontend_logger as logging
+
+CONTROLLER_URL = ""
+
+def submit(
+    meta,
+    image,
+    video,
+    text,
+    num_frames,
+):
+    if meta is None:
+        meta = ConvMeta()
+
+    meta.pop_error()
+
+    check_text = (text != "" and text is not None)
+    check_image = image is not None
+    check_video = video is not None
+
+    if check_text + check_image + check_video != 1:
+        logging.info(f"{meta.log_id}: invalid input: give multi madality simultaneously for single modality input")
+        gr.Error("Invalid input number, must give exactly one modality input at a time")
+        return meta.format_chatbot(), meta, None, None, ""
+
+    if check_text:
+        meta.append(Role.USER, DataMeta.build(text=text))
+    elif check_image:
+        meta.append(Role.USER, DataMeta.build(image=image))
+    elif check_video:
+        frames = extract_frames(video, num_frames)
+        meta.append(Role.USER, DataMeta.build(frames=frames))
+
+    return meta.format_chatbot(), meta, None, None, ""
+
+
+def clear_history(meta):
+    if meta is None:
+        meta = ConvMeta()
+    meta.clear()
+    return meta.format_chatbot(), meta
+
+
+def generate(
+    meta,
+    do_sample,
+    max_new_tokens,
+    temperature,
+    top_k,
+    top_p,
+    length_penalty,
+    num_beams,
+    repetition_penalty,
+):
+    if meta is None:
+        meta = ConvMeta()
+
+    meta.pop_error()
+    meta.pop()
+    prompt = meta.format_chat()
+
+    prompt_list, image_list = [], {}
+    for idx, p in enumerate(prompt):
+        if isinstance(p, Image.Image):
+            key = f"[<IMAGE{idx}>]"
+            prompt_list.append(["IMAGE", key])
+
+            buf = io.BytesIO()
+            p.save(buf, format="PNG")
+            image_list[key] = (key, io.BytesIO(buf.getvalue()), "image/png")
+        else:
+            prompt_list.append(["TEXT", p])
+
+    if len(image_list) == 0:
+        image_list = None
+
+    logging.info(f"{meta.log_id}: construct chat reqeust with prompt {prompt_list}")
+
+    t0 = time.time()
+    try:
+        print(do_sample)
+        rsp = requests.post(
+            CONTROLLER_URL + "/v1/mmc",
+            files=image_list,
+            data={
+                "log_id": meta.log_id,
+                "prompt": json.dumps(prompt_list),
+                "do_sample": do_sample,
+                "max_new_tokens": max_new_tokens,
+                "temperature": temperature,
+                "top_k": top_k,
+                "top_p": top_p,
+                "length_penalty": length_penalty,
+                "num_beams": num_beams,
+                "repetition_penalty": repetition_penalty,
+            },
+        )
+    except:
+        rsp = requests.Response()
+        rsp.status_code = 1099
+    t1 = time.time()
+
+    logging.info(f"{meta.log_id}: get response with status code: {rsp.status_code}, time: {(t1-t0)*1000:.3f}ms")
+
+    if rsp.ok:
+        content = json.loads(rsp.text)
+        if content["code"] == 0:
+            meta.append(Role.ASSISTANT, DataMeta.build(text=content["data"]))
+        else:
+            meta.append(Role.ASSISTANT, DataMeta.build(text=f"GENERATE FAILED: {content['data']}"), is_error=True)
+    else:
+        meta.append(Role.ASSISTANT, DataMeta.build(text=f"GENERATE FAILED: http failed with code {rsp.status_code}"), is_error=True)
+
+    return meta.format_chatbot(), meta
+
+
+def build_chat(args):
+    global CONTROLLER_URL
+    CONTROLLER_URL = args.controller_url
+
+    with gr.Blocks(title="Emu", theme=gr.themes.Default(primary_hue="blue", secondary_hue="blue")) as demo:
+        state = gr.State()
+
+        with gr.Row():
+            with gr.Column(scale=2):
+                with gr.Row():
+                    imagebox = gr.Image(type="pil")
+                with gr.Row():
+                    videobox = gr.Video()
+
+                with gr.Accordion("Parameters", open=True, visible=True) as parameter_row:
+                    do_sample = gr.Checkbox(value=False, label="Do Sample", interactive=True)
+                    max_new_tokens = gr.Slider(minimum=0, maximum=2048, value=512, step=1, interactive=True, label="Max Output Tokens")
+                    temperature = gr.Slider(minimum=0, maximum=1, value=0.7, step=0.05, interactive=True, label="Temperature")
+                    top_k = gr.Slider(minimum=1, maximum=5, value=3, step=1, interactive=True, label="Top K")
+                    top_p = gr.Slider(minimum=0, maximum=1, value=0.9, step=0.05, interactive=True, label="Top P")
+                    length_penalty = gr.Slider(minimum=0, maximum=5, value=3, step=0.1, interactive=True, label="Length Penalty")
+                    num_beams = gr.Slider(minimum=1, maximum=10, value=5, step=1, interactive=True, label="Beam Size")
+                    repetition_penalty = gr.Slider(minimum=1.0, maximum=10.0, value=1.0, step=0.5, interactive=True, label="Repetition Penalty")
+                    num_frames = gr.Number(interactive=True, value=8, maximum=12, label="Num Video Frames")
+
+            with gr.Column(scale=6):
+                chatbot = gr.Chatbot(
+                    elem_id="chatbot",
+                    label="Emu Chatbot",
+                    visible=True,
+                    height=1070,
+                )
+
+                with gr.Row():
+                    with gr.Column(scale=8):
+                        textbox = gr.Textbox(
+                            show_label=False,
+                            placeholder="Enter text and add to prompt",
+                            visible=True,
+                            container=False,
+                        )
+
+                    with gr.Column(scale=1, min_width=60):
+                        add_btn = gr.Button(value="Add")
+
+                with gr.Row(visible=True) as button_row:
+                    # upvote_btn = gr.Button(value="👍 Upvote", interactive=False)
+                    # downvote_btn = gr.Button(value="👎 Downvote", interactive=False)
+                    # flag_btn = gr.Button(value="⚠️ Flag", interactive=False)
+                    # regenerate_btn = gr.Button(value="🔄 Regenerate", interactive=False)
+                    clear_btn = gr.Button(value="🗑️ Clear History")
+                    generate_btn = gr.Button(value="Generate")
+
+
+        clear_btn.click(clear_history, inputs=state, outputs=[chatbot, state])
+        textbox.submit(
+            submit,
+            inputs=[
+                state,
+                imagebox,
+                videobox,
+                textbox,
+                num_frames,
+            ],
+            outputs=[
+                chatbot,
+                state,
+                imagebox,
+                videobox,
+                textbox,
+            ],
+        )
+        add_btn.click(
+            submit,
+            inputs=[
+                state,
+                imagebox,
+                videobox,
+                textbox,
+                num_frames,
+            ],
+            outputs=[
+                chatbot,
+                state,
+                imagebox,
+                videobox,
+                textbox,
+            ],
+        )
+        generate_btn.click(
+            generate,
+            inputs=[
+                state,
+                do_sample,
+                max_new_tokens,
+                temperature,
+                top_k,
+                top_p,
+                length_penalty,
+                num_beams,
+                repetition_penalty,
+            ],
+            outputs=[
+                chatbot,
+                state,
+            ],
+        )
+
+    return demo

demo/generation_frontend.py

@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+# ===================================================
+#
+#    Author        : Fan Zhang
+#    Email         : zhangfan@baai.ac.cn
+#    Institute     : Beijing Academy of Artificial Intelligence (BAAI)
+#    Create On     : 2023-12-11 15:35
+#    Last Modified : 2023-12-19 15:02
+#    File Name     : generation_frontend.py
+#    Description   :
+#
+# ===================================================
+
+import base64
+import json
+import io
+import time
+from PIL import Image
+import requests
+
+import gradio as gr
+
+from emu.constants import EVA_IMAGE_SIZE
+from .meta import ConvMeta, Role, DataMeta
+from .utils import frontend_logger as logging
+
+CONTROLLER_URL = ""
+
+def submit(
+    meta,
+    enable_grd,
+    left,
+    top,
+    right,
+    bottom,
+    image,
+    text,
+):
+    if meta is None:
+        meta = ConvMeta()
+
+    meta.pop_error()
+    if meta.has_gen:
+        meta.clear()
+
+    if enable_grd:
+        if text == "" and image is None:
+            logging.info(f"{meta.log_id}: invalid input: no valid data for grounding input")
+            gr.Error("text or image must be given if enable grounding generation")
+            return meta.format_chatbot(), meta, False, 0, 0, EVA_IMAGE_SIZE, EVA_IMAGE_SIZE, None, ""
+
+        meta.append(Role.USER, DataMeta.build(text=text, image=image, coordinate=[left, top, right, bottom]))
+    elif image is not None and text != "":
+        logging.info(f"{meta.log_id}: invalid input: give text and image simultaneously for single modality input")
+        gr.Error("Do not submit text and image data at the same time!!!")
+        return meta.format_chatbot(), meta, False, 0, 0, EVA_IMAGE_SIZE, EVA_IMAGE_SIZE, None, ""
+    elif image is not None:
+        meta.append(Role.USER, DataMeta.build(image=image))
+    elif text != "":
+        meta.append(Role.USER, DataMeta.build(text=text))
+    return meta.format_chatbot(), meta, False, 0, 0, EVA_IMAGE_SIZE, EVA_IMAGE_SIZE, None, ""
+
+
+def clear_history(meta):
+    if meta is None:
+        meta = ConvMeta()
+    meta.clear()
+    return meta.format_chatbot(), meta
+
+
+def generate(meta, classifier_free_guidance, steps):
+    if meta is None:
+        meta = ConvMeta()
+
+    meta.pop_error()
+    meta.pop()
+    prompt = meta.format_prompt()
+
+    prompt_list, image_list = [], {}
+    for idx, p in enumerate(prompt):
+        if isinstance(p, Image.Image):
+            key = f"[<IMAGE{idx}>]"
+            prompt_list.append(["IMAGE", key])
+
+            buf = io.BytesIO()
+            p.save(buf, format="PNG")
+            image_list[key] = (key, io.BytesIO(buf.getvalue()), "image/png")
+        else:
+            prompt_list.append(["TEXT", p])
+
+
+    if len(image_list) == 0:
+        image_list = None
+
+    logging.info(f"{meta.log_id}: construct generation reqeust with prompt {prompt_list}")
+
+    t0 = time.time()
+    try:
+        rsp = requests.post(
+            CONTROLLER_URL + "/v1/mmg",
+            files=image_list,
+            data={
+                "log_id": meta.log_id,
+                "prompt": json.dumps(prompt_list),
+                "classifier_free_guidance": classifier_free_guidance,
+                "steps": steps,
+            },
+        )
+    except:
+        rsp = requests.Response()
+        rsp.status_code = 1099
+    t1 = time.time()
+
+    logging.info(f"{meta.log_id}: get response with status code: {rsp.status_code}, time: {(t1-t0)*1000:.3f}ms")
+
+    if rsp.ok:
+        content = json.loads(rsp.text)
+        if content["code"] == 0:
+            image = Image.open(io.BytesIO(base64.b64decode(content["data"])))
+            meta.append(Role.ASSISTANT, DataMeta.build(image=image, resize=False))
+        else:
+            meta.append(Role.ASSISTANT, DataMeta.build(text=f"GENERATE FAILED: {content['data']}"))
+    else:
+        meta.append(Role.ASSISTANT, DataMeta.build(text=f"GENERATE FAILED: http failed with code {rsp.status_code}"))
+
+    return meta.format_chatbot(), meta
+
+
+def build_generation(args):
+    global CONTROLLER_URL
+    CONTROLLER_URL = args.controller_url
+
+    with gr.Blocks(title="Emu", theme=gr.themes.Default(primary_hue="blue", secondary_hue="blue")) as demo:
+        state = gr.State()
+
+        with gr.Row():
+            with gr.Column(scale=2):
+                with gr.Row():
+                    imagebox = gr.Image(type="pil")
+
+                with gr.Row():
+                    with gr.Accordion("Grounding Parameters", open=True, visible=True) as grounding_row:
+                        enable_grd = gr.Checkbox(label="Enable")
+                        left = gr.Slider(minimum=0, maximum=EVA_IMAGE_SIZE, value=0, step=1, interactive=True, label="left")
+                        top = gr.Slider(minimum=0, maximum=EVA_IMAGE_SIZE, value=0, step=1, interactive=True, label="top")
+                        right = gr.Slider(minimum=0, maximum=EVA_IMAGE_SIZE, value=EVA_IMAGE_SIZE, step=1, interactive=True, label="right")
+                        bottom = gr.Slider(minimum=0, maximum=EVA_IMAGE_SIZE, value=EVA_IMAGE_SIZE, step=1, interactive=True, label="bottom")
+
+                with gr.Row():
+                    with gr.Accordion("Diffusion Parameters", open=True, visible=True) as parameters_row:
+                        cfg = gr.Slider(minimum=1, maximum=30, value=3, step=0.5, interactive=True, label="classifier free guidance")
+                        steps = gr.Slider(minimum=1, maximum=100, value=50, step=1, interactive=True, label="steps")
+
+            with gr.Column(scale=6):
+                chatbot = gr.Chatbot(
+                    elem_id="chatbot",
+                    label="Emu Chatbot",
+                    visible=True,
+                    height=720,
+                )
+
+                with gr.Row():
+                    with gr.Column(scale=8):
+                        textbox = gr.Textbox(
+                            show_label=False,
+                            placeholder="Enter text and add to prompt",
+                            visible=True,
+                            container=False,
+                        )
+
+                    with gr.Column(scale=1, min_width=60):
+                        add_btn = gr.Button(value="Add")
+
+                with gr.Row(visible=True) as button_row:
+                    # upvote_btn = gr.Button(value="👍 Upvote", interactive=False)
+                    # downvote_btn = gr.Button(value="👎 Downvote", interactive=False)
+                    # regenerate_btn = gr.Button(value="🔄 Regenerate", interactive=False)
+                    clear_btn = gr.Button(value="🗑️ Clear History")
+                    generate_btn = gr.Button(value="Generate")
+
+        clear_btn.click(clear_history, inputs=state, outputs=[chatbot, state])
+
+        textbox.submit(
+            submit,
+            inputs=[
+                state,
+                enable_grd,
+                left,
+                top,
+                right,
+                bottom,
+                imagebox,
+                textbox,
+            ],
+            outputs=[
+                chatbot,
+                state,
+                enable_grd,
+                left,
+                top,
+                right,
+                bottom,
+                imagebox,
+                textbox,
+            ],
+        )
+
+        add_btn.click(
+            submit,
+            inputs=[
+                state,
+                enable_grd,
+                left,
+                top,
+                right,
+                bottom,
+                imagebox,
+                textbox,
+            ],
+            outputs=[
+                chatbot,
+                state,
+                enable_grd,
+                left,
+                top,
+                right,
+                bottom,
+                imagebox,
+                textbox,
+            ],
+        )
+
+        generate_btn.click(
+            generate,
+            inputs=[
+                state,
+                cfg,
+                steps,
+            ],
+            outputs=[
+                chatbot,
+                state,
+            ]
+        )
+
+    return demo

demo/meta.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+
+# ===========================================================================================
+#
+#    Copyright (c) Beijing Academy of Artificial Intelligence (BAAI). All rights reserved.
+#
+#    Author        : Fan Zhang
+#    Email         : zhangfan@baai.ac.cn
+#    Institute     : Beijing Academy of Artificial Intelligence (BAAI)
+#    Create On     : 2023-12-12 02:54
+#    Last Modified : 2023-12-19 15:00
+#    File Name     : meta.py
+#    Description   :
+#
+# ===========================================================================================
+
+import base64
+from dataclasses import dataclass, field
+import io
+from enum import Enum
+from PIL import Image
+from typing import List, Tuple
+
+import cv2
+import numpy as np
+
+from emu.constants import EVA_IMAGE_SIZE, GRD_SYMBOL, BOP_SYMBOL, EOP_SYMBOL, BOO_SYMBOL, EOO_SYMBOL
+from emu.constants import DEFAULT_VIDEO_TOKEN, DEFAULT_EOS_TOKEN, USER_TOKEN, ASSISTANT_TOKEN, FAKE_VIDEO_END_TOKEN
+
+from .utils import gen_id, frontend_logger as logging
+
+
+class Role(Enum):
+    UNKNOWN = 0,
+    USER = 1,
+    ASSISTANT = 2,
+
+
+class DataType(Enum):
+    UNKNOWN = 0,
+    TEXT = 1,
+    IMAGE = 2,
+    GROUNDING = 3,
+    VIDEO = 4,
+    ERROR = 5,
+
+
+@dataclass
+class DataMeta:
+    datatype: DataType = DataType.UNKNOWN
+    text: str = None
+    image: Image.Image = None
+    mask: Image.Image = None
+    coordinate: List[int] = None
+    frames: List[Image.Image] = None
+    stack_frame: Image.Image = None
+
+    @property
+    def grounding(self):
+        return self.coordinate is not None
+
+    @property
+    def text_str(self):
+        return self.text
+
+    @property
+    def image_str(self):
+        return self.image2str(self.image)
+
+    @property
+    def video_str(self):
+        ret = f'<div style="overflow:scroll"><b>[VIDEO]</b></div>{self.image2str(self.stack_frame)}'
+        return ret
+
+    @property
+    def grounding_str(self):
+        ret = ""
+        if self.text is not None:
+            ret += f'<div style="overflow:scroll"><b>[PHRASE]</b>{self.text}</div>'
+
+        ret += self.image2str(self.mask)
+
+        if self.image is not None:
+            ret += self.image2str(self.image)
+        return ret
+
+    def image2str(self, image):
+        buf = io.BytesIO()
+        image.save(buf, format="WEBP")
+        i_str = base64.b64encode(buf.getvalue()).decode()
+        return f'<div style="float:left"><img src="data:image/png;base64, {i_str}"></div>'
+
+    def format_chatbot(self):
+        match self.datatype:
+            case DataType.TEXT:
+                return self.text_str
+            case DataType.IMAGE:
+                return self.image_str
+            case DataType.VIDEO:
+                return self.video_str
+            case DataType.GROUNDING:
+                return self.grounding_str
+            case _:
+                return ""
+
+    def format_prompt(self) -> List[str | Image.Image]:
+        match self.datatype:
+            case DataType.TEXT:
+                return [self.text]
+            case DataType.IMAGE:
+                return [self.image]
+            case DataType.VIDEO:
+                return [DEFAULT_VIDEO_TOKEN] + self.frames + [FAKE_VIDEO_END_TOKEN]
+            case DataType.GROUNDING:
+                ret = []
+                if self.text is not None:
+                    ret.append(f"{BOP_SYMBOL}{self.text}{EOP_SYMBOL}")
+                ret += [BOO_SYMBOL, self.mask, EOO_SYMBOL]
+                if self.image is not None:
+                    ret.append(self.image)
+                return ret
+            case _:
+                return []
+
+    def __str__(self):
+        s = ""
+        if self.text is not None:
+            s += f"T:{self.text}"
+
+        if self.image is not None:
+            w, h = self.image.size
+            s += f"[I:{h}x{w}]"
+
+        if self.coordinate is not None:
+            l, t, r, b = self.coordinate
+            s += f"[C:({l:03d},{t:03d}),({r:03d},{b:03d})]"
+
+        if self.frames is not None:
+            w, h = self.frames[0].size
+            s += f"[V:{len(self.frames)}x{h}x{w}]"
+
+        return s
+
+    @classmethod
+    def build(cls, text=None, image=None, coordinate=None, frames=None, is_error=False, *, resize: bool = True):
+        ins = cls()
+        ins.text = text if text != "" else None
+        ins.image = cls.resize(image, force=resize)
+        # ins.image = image
+        ins.coordinate = cls.fix(coordinate)
+        ins.frames = cls.resize(frames, force=resize)
+        # ins.frames = frames
+
+        if is_error:
+            ins.datatype = DataType.ERROR
+        elif coordinate is not None:
+            ins.datatype = DataType.GROUNDING
+            ins.draw_box()
+        elif image is not None:
+            ins.datatype = DataType.IMAGE
+        elif text is not None:
+            ins.datatype = DataType.TEXT
+        else:
+            ins.datatype = DataType.VIDEO
+            ins.stack()
+
+        return ins
+
+    @classmethod
+    def fix(cls, coordinate):
+        if coordinate is None:
+            return None
+
+        l, t, r, b = coordinate
+        l = min(EVA_IMAGE_SIZE, max(0, l))
+        t = min(EVA_IMAGE_SIZE, max(0, t))
+        r = min(EVA_IMAGE_SIZE, max(0, r))
+        b = min(EVA_IMAGE_SIZE, max(0, b))
+        return min(l, r), min(t, b), max(l, r), max(t, b)
+
+    @classmethod
+    def resize(cls, image: Image.Image | List[Image.Image] | None, *, force: bool = True):
+        if image is None:
+            return None
+
+        if not force:
+            return image
+
+        if isinstance(image, Image.Image):
+            image = [image]
+
+        for idx, im in enumerate(image):
+            w, h = im.size
+            if w < h:
+                h = int(EVA_IMAGE_SIZE / w * h)
+                w = EVA_IMAGE_SIZE
+            else:
+                w = int(EVA_IMAGE_SIZE / h * w)
+                h = EVA_IMAGE_SIZE
+
+            image[idx] = im.resize((w, h))
+
+        return image if len(image) > 1 else image[0]
+
+    def draw_box(self):
+        left, top, right, bottom = self.coordinate
+        mask = np.zeros((EVA_IMAGE_SIZE, EVA_IMAGE_SIZE, 3), dtype=np.uint8)
+        mask = cv2.rectangle(mask, (left, top), (right, bottom), (255, 255, 255), 3)
+        self.mask = Image.fromarray(mask)
+
+    def stack(self):
+        w, h = self.frames[0].size
+        n = len(self.frames)
+        stack_frame = Image.new(mode="RGB", size=(w*n, h))
+        for idx, f in enumerate(self.frames):
+            stack_frame.paste(f, (idx*w, 0))
+        self.stack_frame = stack_frame
+
+
+class ConvMeta:
+
+    def __init__(self):
+        self.system: str = "You are a helpful assistant, dedicated to delivering comprehensive and meticulous responses."
+        self.message: List[Tuple[Role, DataMeta]] = []
+        self.log_id: str = gen_id()
+
+        logging.info(f"{self.log_id}: create new round of chat")
+
+    def append(self, r: Role, p: DataMeta):
+        logging.info(f"{self.log_id}: APPEND [{r.name}] prompt element, type: {p.datatype.name}, message: {p}")
+        self.message.append((r, p))
+
+    def format_chatbot(self):
+        ret = []
+        for r, p in self.message:
+            cur_p = p.format_chatbot()
+            if r == Role.USER:
+                ret.append((cur_p, None))
+            else:
+                ret.append((None, cur_p))
+        return ret
+
+    def format_prompt(self):
+        ret = []
+        has_coor = False
+        for _, p in self.message:
+            has_coor |= (p.datatype == DataType.GROUNDING)
+            ret += p.format_prompt()
+
+        if has_coor:
+            ret.insert(0, GRD_SYMBOL)
+
+        logging.info(f"{self.log_id}: format generation prompt: {ret}")
+        return ret
+
+    def format_chat(self):
+        ret = [self.system]
+
+        prev_r = None
+        for r, p in self.message:
+            if prev_r != r:
+                if prev_r == Role.ASSISTANT:
+                    ret.append(f"{DEFAULT_EOS_TOKEN}{USER_TOKEN}: ")
+                elif prev_r is None:
+                    ret.append(f" {USER_TOKEN}: ")
+                else:
+                    ret.append(f" {ASSISTANT_TOKEN}: ")
+                ret += p.format_prompt()
+                prev_r = r
+            else:
+                ret += p.format_prompt()
+
+        ret.append(f" {ASSISTANT_TOKEN}:")
+
+        logging.info(f"{self.log_id}: format chat prompt: {ret}")
+        return ret
+
+    def clear(self):
+        logging.info(f"{self.log_id}: clear chat history, end current chat round.")
+        del self.message
+        self.message = []
+        self.log_id = gen_id()
+
+    def pop(self):
+        if self.has_gen:
+            logging.info(f"{self.log_id}: pop out previous generation / chat result")
+            self.message.pop()
+
+    def pop_error(self):
+        self.message = [(r, p) for r, p in self.message if p.datatype != DataType.ERROR]
+
+    @property
+    def has_gen(self):
+        if len(self.message) == 0:
+            return False
+        if self.message[-1][0] == Role.USER:
+            return False
+        return True

demo/utils.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+
+# =================================================== 
+#
+#    Author        : Fan Zhang
+#    Email         : zhangfan@baai.ac.cn
+#    Institute     : Beijing Academy of Artificial Intelligence (BAAI)
+#    Create On     : 2023-12-13 09:48
+#    Last Modified : 2023-12-14 01:53
+#    File Name     : utils.py
+#    Description   : 
+#
+# =================================================== 
+
+from datetime import datetime
+import logging
+import logging.config
+import hashlib
+import os.path as osp
+import uuid
+from PIL import Image
+
+from decord import VideoReader
+
+
+def config_logger(logger_name):
+    logger_config = {
+        "version": 1,
+        "formatters": {
+            "standard": {
+                "format": "%(asctime)s - %(filename)s: %(lineno)d - [%(levelname)s] - %(message)s",
+                "datefmt": "%Y-%m-%d %H:%M:%S",
+            },
+        },
+        "handlers": {
+            "console": {
+                "class": "logging.StreamHandler",
+                "formatter": "standard",
+                "level": "INFO",
+            },
+            "file": {
+                "class": "logging.handlers.TimedRotatingFileHandler",
+                "filename": osp.join(osp.dirname(__file__), "..", "log", f"{logger_name}.log"),
+                "formatter": "standard",
+                "level": "INFO",
+                "when": "D",
+                "interval": 7,
+                "backupCount": 90,
+            },
+        },
+        "loggers": {
+            logger_name: {
+                "handlers": ["file", "console"],
+                "level": "INFO",
+                "propagate": True,
+            },
+        },
+    }
+
+    logging.config.dictConfig(logger_config)
+    logger = logging.getLogger(logger_name)
+    return logger
+
+frontend_logger = config_logger("Emu-v2_frontend")
+beckend_logger = config_logger("Emu-v2_backend")
+
+
+def extract_frames(video, num_frames):
+    video = VideoReader(video)
+    total_frames = len(video)
+    segment = int(total_frames // num_frames)
+
+    frames = video.get_batch(list(range(int(segment//2), total_frames, segment))).asnumpy()
+    frames = [Image.fromarray(f) for f in frames]
+    return frames
+
+
+def image2md5(image: Image.Image):
+    md5hash = hashlib.md5(image.tobytes())
+    return md5hash.hexdigest()
+
+
+def gen_id():
+    logid = datetime.now().strftime("%Y%m%d%H%M%d")
+    logid += f"{uuid.uuid4().hex}"
+    return logid
+

requirements.txt

@@ -0,0 +1,5 @@
+gradio
+pillow
+numpy
+opencv-python
+decord