update

.gitignore

@@ -0,0 +1 @@
+.idea

README.md

@@ -1,14 +1,13 @@
 ---
-title: MotionBench
-emoji: 🐠
-colorFrom: yellow
-colorTo: yellow
+title: MotionBench Leaderboard
+emoji: 🐨
+colorFrom: purple
+colorTo: blue
 sdk: gradio
-sdk_version: 5.6.0
+sdk_version: 4.36.1
 app_file: app.py
 pinned: false
-license: apache-2.0
-short_description: Leaderboard for MotionBench
+license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,247 @@
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+
+import gradio as gr
+import pandas as pd
+import json
+
+from constants import *
+from huggingface_hub import Repository
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+
+global data_component, filter_component
+
+
+def download_csv():
+    # pull the results and return this file!
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN,
+                                 repo_type="dataset")
+    submission_repo.git_pull()
+    return CSV_DIR, gr.update(visible=True)
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+
+def add_new_eval(
+        input_file,
+        model_name_textbox: str,
+        revision_name_textbox: str,
+        model_link: str,
+        model_date:str,
+        LLM_type: str,
+        LLM_name_textbox: str,
+):
+    if input_file is None:
+        return "Error! Empty file!"
+
+    upload_data = json.loads(input_file)
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN,
+                                 repo_type="dataset",git_user="auto-uploader",git_email="uploader@163.com")
+    submission_repo.git_pull()
+    csv_data = pd.read_csv(CSV_DIR)
+
+    if LLM_type == 'Other':
+        LLM_name = LLM_name_textbox
+    else:
+        LLM_name = LLM_type
+
+    if revision_name_textbox == '':
+        col = csv_data.shape[0]
+        model_name = model_name_textbox
+    else:
+        model_name = revision_name_textbox
+        model_name_list = csv_data['Model']
+        name_list = [name.split(']')[0][1:] for name in model_name_list]
+        if revision_name_textbox not in name_list:
+            col = csv_data.shape[0]
+        else:
+            col = name_list.index(revision_name_textbox)
+
+    if model_link == '' or "](" in model_name:
+        model_name = model_name  # no url
+    else:
+        model_name = '[' + model_name + '](' + model_link + ')'
+
+    # add new data
+    new_data = [
+        model_name,
+        LLM_name,
+        model_date,
+        model_link
+    ]
+    for key in TASK_INFO:
+        if key in upload_data:
+            new_data.append(round(100*upload_data[key],1))
+        else:
+            new_data.append(0)
+    # print(new_data)
+    # print(csv_data.loc[col-1])
+    csv_data.loc[col] = new_data
+    csv_data = csv_data.to_csv(CSV_DIR, index=False)
+    submission_repo.push_to_hub()
+    return 0
+
+
+def get_baseline_df():
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN,
+                                 repo_type="dataset")
+    submission_repo.git_pull()
+    df = pd.read_csv(CSV_DIR)
+    df = df.sort_values(by="Overall", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group.value
+    df = df[present_columns]
+    return df
+
+
+def get_all_df():
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN,
+                                 repo_type="dataset")
+    submission_repo.git_pull()
+    df = pd.read_csv(CSV_DIR)
+    df = df.sort_values(by="Overall", ascending=False)
+    return df
+
+
+def on_filter_model_size_method_change(selected_columns):
+    updated_data = get_all_df()
+
+    # columns:
+    selected_columns = [item for item in TASK_INFO if item in selected_columns]
+    present_columns = MODEL_INFO + selected_columns
+    # print("selected_columns",'|'.join(selected_columns))
+    updated_data = updated_data[present_columns]
+    updated_data = updated_data.sort_values(by=selected_columns[0], ascending=False)
+    updated_headers = present_columns
+    update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+    # print(updated_data,present_columns,update_datatype)
+    filter_component = gr.components.Dataframe(
+        value=updated_data,
+        headers=updated_headers,
+        type="pandas",
+        datatype=update_datatype,
+        interactive=False,
+        visible=True,
+    )
+
+    return filter_component  # .value
+
+
+block = gr.Blocks()
+with block:
+    gr.Markdown(
+        LEADERBORAD_INTRODUCTION
+    )
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 MotionBench", elem_id="lvbench-tab-table", id=1):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        lines=10,
+                    )
+
+            gr.Markdown(
+                TABLE_INTRODUCTION
+            )
+
+            # selection for column part:
+            checkbox_group = gr.CheckboxGroup(
+                choices=TASK_INFO,
+                value=AVG_INFO,
+                label="Evaluation Dimension",
+                interactive=True,
+            )
+
+            data_component = gr.components.Dataframe(
+                value=get_baseline_df,
+                headers=COLUMN_NAMES,
+                type="pandas",
+                datatype=DATA_TITILE_TYPE,
+                interactive=False,
+                visible=True,
+            )
+
+            checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[checkbox_group],
+                                  outputs=data_component)
+
+        # table 2
+        with gr.TabItem("📝 About", elem_id="lvbench-tab-table", id=2):
+            gr.Markdown(LEADERBORAD_INFO, elem_classes="markdown-text")
+
+        # table 3 
+        with gr.TabItem("🚀 Submit here! ", elem_id="lvbench-tab-table", id=3):
+            gr.Markdown(LEADERBORAD_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(
+                        label="Model name", placeholder="CogVLM2-Video"
+                    )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name", placeholder="CogVLM2-Video"
+                    )
+
+                with gr.Column():
+                    LLM_type = gr.Dropdown(
+                        choices=["LLaMA-3-8B", "Vicuna-7B", "Flan-T5-XL", "LLaMA-7B", "InternLM-7B", "Other"],
+                        label="LLM type",
+                        multiselect=False,
+                        value="LLaMA-3-8B",
+                        interactive=True,
+                    )
+                    LLM_name_textbox = gr.Textbox(
+                        label="LLM model (for Other)",
+                        placeholder="LLaMA-3-8B"
+                    )
+                    model_link = gr.Textbox(
+                        label="Model Link", placeholder="https://cogvlm2-video.github.io/"
+                    )
+                    model_date = gr.Textbox(
+                        label="Model Date", placeholder="2024/8/22"
+                    )
+
+
+            with gr.Column():
+                input_file = gr.components.File(label="Click to Upload a json File", file_count="single", type='binary')
+                submit_button = gr.Button("Submit Eval")
+
+                submission_result = gr.Markdown()
+                submit_button.click(
+                    add_new_eval,
+                    inputs=[
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        model_link,
+                        model_date,
+                        LLM_type,
+                        LLM_name_textbox,
+                    ],
+                )
+
+
+    def refresh_data():
+        value1 = get_baseline_df()
+        return value1
+
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+    with gr.Row():
+        result_download = gr.Button("Download Leaderboard")
+        file_download = gr.File(label="download the csv of leaderborad.", visible=False)
+        data_run.click(on_filter_model_size_method_change, inputs=[checkbox_group], outputs=data_component)
+        result_download.click(download_csv, inputs=None, outputs=[file_download, file_download])
+
+block.launch()

constants.py

@@ -0,0 +1,39 @@
+import os
+
+# this is .py for store constants
+MODEL_INFO = ["Model", "Language Model", "Date"]
+TASK_INFO = ["Dev Avg", "Test Avg", "MR", "LM", "CM", "MO", "AO", "RC"]
+AVG_INFO = ["Dev Avg"]
+
+DATA_TITILE_TYPE = ['markdown', 'markdown', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number',
+                    'number', 'number']
+
+SUBMISSION_NAME = "MotionBench_submission"
+SUBMISSION_URL = os.path.join("https://huggingface.co/datasets/THUDM/", SUBMISSION_NAME)
+CSV_DIR = "./MotionBench_submission/result.csv"
+
+COLUMN_NAMES = MODEL_INFO + TASK_INFO
+
+LEADERBORAD_INTRODUCTION = """# MotionBench Leaderboard
+    Welcome to the leaderboard of the MotionBench! 🏆 You can prepare your submission by following the [instructions](https://github.com/THUDM/MotionBench?tab=readme-ov-file#get-evaluation-results-and-submit-to-leaderboard).
+    """
+
+SUBMIT_INTRODUCTION = """# Submit on MotionBench Benchmark Introduction
+"""
+
+TABLE_INTRODUCTION = """
+    """
+
+LEADERBORAD_INFO = """
+      In recent years, vision language models (VLMs) have made significant advancements in video understanding. However, a crucial capability — fine-grained motion comprehension — remains under-explored in current benchmarks. To address this gap, we propose MotionBench, a comprehensive evaluation benchmark designed to assess the fine-grained motion comprehension of video understanding models. MotionBench evaluates models' motion-level perception through six primary categories of motion-oriented question types and includes data collected from diverse sources, ensuring a broad representation of real-world video content. Experimental results reveal that existing VLMs perform poorly in understanding fine-grained motions. To enhance VLM's ability to perceive fine-grained motion within a limited sequence length of LLM, we conduct extensive experiments reviewing VLM architectures optimized for video feature compression and propose a novel and efficient Through-Encoder (TE) Fusion method. Experiments show that higher frame rate inputs and TE Fusion yield improvements in motion understanding, yet there is still substantial room for enhancement. Our benchmark aims to guide and motivate the development of more capable video understanding models, emphasizing the importance of fine-grained motion comprehension.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{xxx,
+      title={MotionBench: Benchmarking and Improving Fine-grained Video Motion Understanding for Vision Language Models},
+      author={xxx},
+      year={2024},
+      eprint={xx.xx},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV}
+}"""

requirements.txt

@@ -0,0 +1,16 @@
+APScheduler
+black
+datasets
+gradio
+gradio[oauth]
+gradio_leaderboard==0.0.9
+gradio_client
+huggingface-hub>=0.18.0
+matplotlib
+numpy
+pandas
+python-dateutil
+tqdm
+transformers
+tokenizers>=0.15.0
+sentencepiece