first

README.md

@@ -1,13 +1,13 @@
 ---
-title: Video Bench
-emoji: 😻
-colorFrom: blue
-colorTo: yellow
+title: Video-Bench Leaderboard
+emoji: 🏆
+colorFrom: gray
+colorTo: blue
 sdk: gradio
-sdk_version: 4.7.1
+sdk_version: 3.40.1
 app_file: app.py
 pinned: false
-license: apache-2.0
+license: cc-by-4.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,243 @@
+
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+
+import gradio as gr
+import pandas as pd
+import json
+import pdb
+import tempfile
+
+from constants import *
+from src.auto_leaderboard.model_metadata_type import ModelType
+
+global data_component, filter_component
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+def add_new_eval(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    model_type: str,
+    model_link: str,
+    LLM_type: str,
+    LLM_name_textbox: str,
+):
+    if input_file is None:
+        return "Error! Empty file!"
+    else:
+        input_data = input_file.decode("utf-8").split('\n')[1].split(',')
+        input_data = [str(i) for i in input_data]
+
+        csv_data = pd.read_csv(CSV_DIR)
+
+
+        if LLM_type == 'Other':
+            LLM_name = LLM_name_textbox
+        else:
+            LLM_name = LLM_type
+        
+        if revision_name_textbox == '':
+            col = csv_data.shape[0]
+            model_name = model_name_textbox
+        else:
+            model_name = revision_name_textbox
+            model_name_list = csv_data['Model']
+            name_list = [name.split(']')[0][1:] for name in model_name_list]
+            if revision_name_textbox not in name_list:
+                col = csv_data.shape[0]
+            else:
+                col = name_list.index(revision_name_textbox)    
+
+        if model_link == '':
+            model_name = model_name  # no url
+        else:
+            model_name = '[' + model_name + '](' + model_link + ')'
+
+        # add new data
+        new_data = [
+            model_type, 
+            model_name, 
+            LLM_name,
+            input_data[0],
+            input_data[1],
+            input_data[2],
+            input_data[3],
+            input_data[4],
+            input_data[5],
+            input_data[6],
+            input_data[7],
+            input_data[8],
+            input_data[9],
+            input_data[10],
+            input_data[11],
+            input_data[12],
+            input_data[13],
+            input_data[14],
+            input_data[15],
+            input_data[16],
+            ]
+        csv_data.loc[col] = new_data
+        csv_data = csv_data.to_csv(CSV_DIR, index=False)
+    return 0
+
+def get_baseline_df():
+    # pdb.set_trace()
+    df = pd.read_csv(CSV_DIR, dtype=str)
+    df = df.sort_values(by="Avg. All", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group.value
+    df = df[present_columns]
+    return df
+
+def get_all_df():
+    df = pd.read_csv(CSV_DIR, dtype=str)
+    df = df.sort_values(by="Avg. All", ascending=False)
+    return df
+
+block = gr.Blocks()
+
+
+with block:
+    gr.Markdown(
+        LEADERBORAD_INTRODUCTION
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 Video Benchmark", elem_id="video-benchmark-tab-table", id=0):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                    ).style(show_copy_button=True)
+    
+            gr.Markdown(
+                TABLE_INTRODUCTION
+            )
+
+            # selection for column part:
+            checkbox_group = gr.CheckboxGroup(
+                choices=TASK_INFO_v2,
+                value=AVG_INFO,
+                label="Select options",
+                interactive=True,
+            )
+
+            # 创建数据帧组件
+            data_component = gr.components.Dataframe(
+                value=get_baseline_df, 
+                headers=COLUMN_NAMES,
+                type="pandas", 
+                datatype=DATA_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+                )
+    
+            def on_checkbox_group_change(selected_columns):
+                # pdb.set_trace()
+                selected_columns = [item for item in TASK_INFO_v2 if item in selected_columns]
+                present_columns = MODEL_INFO + selected_columns
+                updated_data = get_all_df()[present_columns]
+                updated_data = updated_data.sort_values(by=present_columns[3], ascending=False)
+                updated_headers = present_columns
+                update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+
+                filter_component = gr.components.Dataframe(
+                    value=updated_data, 
+                    headers=updated_headers,
+                    type="pandas", 
+                    datatype=update_datatype,
+                    interactive=False,
+                    visible=True,
+                    )
+                # pdb.set_trace()
+        
+                return filter_component.value
+
+            # 将复选框组关联到处理函数
+            checkbox_group.change(fn=on_checkbox_group_change, inputs=checkbox_group, outputs=data_component)
+        '''
+        # table 2
+        with gr.TabItem("📝 About", elem_id="seed-benchmark-tab-table", id=2):
+            gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
+        '''
+        # table 3 
+        with gr.TabItem("🚀 Submit here! ", elem_id="seed-benchmark-tab-table", id=3):
+            gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(
+                        label="Model name", placeholder="LLaMA-7B"
+                        )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name", placeholder="LLaMA-7B"
+                    )
+                    model_type = gr.Dropdown(
+                        choices=[                         
+                            "LLM",
+                            "ImageLLM",
+                            "VideoLLM",
+                            "Other", 
+                        ], 
+                        label="Model type", 
+                        multiselect=False,
+                        value="ImageLLM",
+                        interactive=True,
+                    )
+
+                with gr.Column():
+                    
+                    LLM_type = gr.Dropdown(
+                        choices=["Vicuna-7B", "Flan-T5-XL", "LLaMA-7B", "Other"],
+                        label="LLM type", 
+                        multiselect=False,
+                        value="LLaMA-7B",
+                        interactive=True,
+                    )
+                    LLM_name_textbox = gr.Textbox(
+                        label="LLM model (for Other)",
+                        placeholder="LLaMA-13B"
+                    )
+                    model_link = gr.Textbox(
+                        label="Model Link", placeholder="https://huggingface.co/decapoda-research/llama-7b-hf"
+                    )
+
+            with gr.Column():
+
+                input_file = gr.File(label="Click to Upload a csv File", type='binary')
+                submit_button = gr.Button("Submit Eval")
+    
+                submission_result = gr.Markdown()
+                submit_button.click(
+                    add_new_eval,
+                    inputs=[
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        model_type,
+                        model_link,
+                        LLM_type,
+                        LLM_name_textbox,
+                    ],
+                    # outputs = submission_result,
+                )
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(
+            get_baseline_df, outputs=data_component
+        )
+
+    # block.load(get_baseline_df, outputs=data_title)
+
+block.launch()

constants.py

@@ -0,0 +1,75 @@
+# this is .py for store constants 
+MODEL_INFO = ["Model Type", "Model", "Language Model"]
+
+TASK_INFO_v2 = ["Avg. All", "Avg. Video-Exclusive", "Avg. Prior-Knowledge QA", "Avg. Decision-Making",
+                "ActivityNet", "MSVD", "MSRVTT", "TGIF", "Youcook2", "Ucfcrime", "MOT",
+                "TVQA", "MV", "NBA",
+                "Driving-exam", "Driving-decision-making", "SQA3D"]
+
+AVG_INFO = ["Avg. All", "Avg. Video-Exclusive", "Avg. Prior-Knowledge QA", "Avg. Decision-Making"]
+DATA_TITILE_TYPE = ["markdown", "markdown", "markdown", "markdown",
+                    "number", "number", "number", "number", "number", "number", "number",
+                    "number", "number", "number",
+                    "number", "number", "number"]
+CSV_DIR = "./file/result.csv"
+
+# COLUMN_NAMES = MODEL_INFO + TASK_INFO
+COLUMN_NAMES = MODEL_INFO + TASK_INFO_v2
+
+
+LEADERBORAD_INTRODUCTION = """# Video-Bench Leaderboard
+    
+    Welcome to the leaderboard of the Video-Bench! 🏆 
+    Video-Bench consists of 15K questions with human-like video for evaluating Video-LLMs, covering three-level and 13 evaluation dimensions including both the spatial and temporal understanding.
+    Please refer to [our paper](https://arxiv.org/abs/2311.16103) for more details.
+    """
+
+SUBMIT_INTRODUCTION = """# Submit Introduction
+    1. Obtain JSON file from our [github repository](https://github.com/AILab-CVC/SEED-Bench#leaderboard-submit) after evaluation. For example, you can obtain InstructBLIP's JSON file as results/results.json after running 
+    ```shell
+    python eval.py --model instruct_blip --anno_path SEED-Bench.json --output-dir results
+    ```
+    2. If you want to update model performance by uploading new results, please ensure 'Model Name Revision' is the same as what's shown in the leaderboard. For example, if you want to modify InstructBLIP's performance, you need to fill in 'InstructBLIP' in 'Revision Model Name'.
+    3. Please provide the correct link of your model's repository for each submission.  
+    4. For the evaluation dimension, you can choose "All/Image/Video", and the results of dimensions that are not evaluated will be set to zero.
+    5. After clicking 'Submit Eval', you can click 'Refresh' to obtain the latest result in the leaderboard.
+
+    ## Submit Example
+    For example, if you want to upload InstructBLIP's result in the leaderboard, you need to:
+    1. Fill in 'InstructBLIP' in 'Model Name' if it is your first time to submit your result (You can leave 'Revision Model Name' blank).
+    2. Fill in 'InstructBLIP' in 'Revision Model Name' if you want to update your result (You can leave 'Model Name' blank).
+    2. Select 'ImageLLM' in 'Model Type'.
+    3. Fill in 'https://github.com/salesforce/LAVIS' in 'Model Link'.
+    4. Select 'Flan-T5-XL' in 'LLM Type'.
+    5. Select 'All' in 'Evaluation Dimension'.
+    6. Upload results.json.
+    7. Click the 'Submit Eval' button.
+    8. Click 'Refresh' to obtain the uploaded leaderboard.
+"""
+
+TABLE_INTRODUCTION = """In the table below, we summarize each task performance of all the models.
+        We use accurancy(%) as the primary evaluation metric for each tasks. 
+    """
+
+LEADERBORAD_INFO = """
+      Based on powerful Large Language Models (LLMs), recent generative Multimodal Large Language Models (MLLMs) have gained prominence as a pivotal research area, exhibiting remarkable capability for both comprehension and generation. 
+      In this work, we address the evaluation of generative comprehension in MLLMs as a preliminary step towards a comprehensive assessment of generative models, by introducing a benchmark named SEED-Bench. 
+      SEED-Bench consists of 19K multiple choice questions with accurate human annotations (x6 larger than existing benchmarks), which spans 12 evaluation dimensions including the comprehension of both the image and video modality. 
+      We develop an advanced pipeline for generating multiple-choice questions that target specific evaluation dimensions, integrating both automatic filtering and manual verification processes. 
+      Multiple-choice questions with groundtruth options derived from human annotation enables an objective and efficient assessment of model performance, eliminating the need for human or GPT intervention during evaluation. 
+      We further evaluate the performance of 18 models across all 12 dimensions, covering both the spatial and temporal understanding. 
+      By revealing the limitations of existing MLLMs through evaluation results, we aim for SEED-Bench to provide insights for motivating future research.
+"""
+
+
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{ning2023videobench,
+      title={Video-Bench: A Comprehensive Benchmark and Toolkit for Evaluating Video-based Large Language Models}, 
+      author={Munan Ning and Bin Zhu and Yujia Xie and Bin Lin and Jiaxi Cui and Lu Yuan and Dongdong Chen and Li Yuan},
+      year={2023},
+      eprint={2311.16103},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV}
+}
+}"""

file/result.csv

@@ -0,0 +1,2 @@
+Model Type,Model,Language Model,Avg. All,Avg. Video-Exclusive,Avg. Prior-Knowledge QA,Avg. Decision-Making,ActivityNet,MSVD,MSRVTT,TGIF,Youcook2,Ucfcrime,MOT,TVQA,MV,NBA,Driving-exam,Driving-decision-making,SQA3D
+LLM,[Flan-T5](https://huggingface.co/google/flan-t5-xl),Flan-T5-XL,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1

file/sample_to_upload.csv

@@ -0,0 +1,2 @@
+Avg. All,Avg. Video-Exclusive,Avg. Prior-Knowledge QA,Avg. Decision-Making,ActivityNet,MSVD,MSRVTT,TGIF,Youcook2,Ucfcrime,MOT,TVQA,MV,NBA,Driving-exam,Driving-decision-making,SQA3D
+4,2,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1

requirements.txt

@@ -0,0 +1,70 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+APScheduler==3.10.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.11.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+gradio==3.27.0
+gradio_client==0.1.3
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.2
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.0
+Pillow==9.5.0
+plotly==5.14.1
+pyarrow==11.0.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+requests==2.28.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+transformers==4.28.1
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+yarl==1.8.2

src/auto_leaderboard/model_metadata_type.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from enum import Enum
+import glob
+import json
+import os
+from typing import Dict, List
+
+from ..utils_display import AutoEvalColumn
+
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str # emoji
+
+model_type_symbols = {
+    "LLM": "🟢",
+    "ImageLLM": "🔶",
+    "VideoLLM": "⭕",
+    "Other": "🟦",
+}
+
+class ModelType(Enum):
+    PT = ModelInfo(name="LLM", symbol="🟢")
+    FT = ModelInfo(name="ImageLLM", symbol="🔶")
+    IFT = ModelInfo(name="VideoLLM", symbol="⭕")
+    RL = ModelInfo(name="Other", symbol="🟦")
+
+    def to_str(self, separator = " "):
+        return f"{self.value.symbol}{separator}{self.value.name}" 
+

src/utils_display.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+    displayed_by_default: bool 
+    hidden: bool = False
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model_type_symbol = ColumnContent("T", "str", True)
+    model = ColumnContent("Model", "markdown", True)
+    average = ColumnContent("Average ⬆️", "number", True)
+    arc = ColumnContent("ARC", "number", True)
+    hellaswag = ColumnContent("HellaSwag", "number", True)
+    mmlu = ColumnContent("MMLU", "number", True)
+    truthfulqa = ColumnContent("TruthfulQA", "number", True)
+    model_type = ColumnContent("Type", "str", False)
+    precision = ColumnContent("Precision", "str", False, True)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    revision = ColumnContent("Model sha", "str", False, False)
+    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
+
+@dataclass(frozen=True)
+class EloEvalColumn: # Elo evals column
+    model = ColumnContent("Model", "markdown", True)
+    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+    human_all = ColumnContent("Human (all)", "number", True)
+    human_instruct = ColumnContent("Human (instruct)", "number", True)
+    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn: # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "bool", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]
+
+
+KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+MODEL_PAGE = "https://huggingface.co/models"
+LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+
+    if model_name in LLAMAS:
+        link = LLAMA_LINK
+        model_name = model_name.split("/")[1]
+    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+        link = VICUNA_LINK
+        model_name = "stable-vicuna-13b"
+    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+        link = ALPACA_LINK
+        model_name = "alpaca-13b"
+    if model_name == "dolly-12b":
+        link = DOLLY_LINK
+    elif model_name == "vicuna-13b":
+        link = VICUNA_LINK
+    elif model_name == "koala-13b":
+        link = KOALA_LINK
+    elif model_name == "oasst-12b":
+        link = OASST_LINK
+    #else:
+    #    link = MODEL_PAGE
+  
+    return model_hyperlink(link, model_name)
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"