v2 update

app.py

@@ -1,4 +1,5 @@
 from utils import *
+from labels_v2 import *
 
 global data_component
 
@@ -15,7 +16,7 @@ with gr.Blocks() as block:
     
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         # Table 1
-        with gr.TabItem("📊 MMEB", elem_id="qa-tab-table1", id=1):
+        with gr.TabItem("📊 MMEB-V1", elem_id="qa-tab-table1", id=1):
             with gr.Row():
                 with gr.Accordion("Citation", open=False):
                     citation_button = gr.Textbox(
@@ -52,7 +53,7 @@ with gr.Blocks() as block:
                     label="Maximum number of parameters (B)",
                 )
 
-            task_choices = [col for col in COLUMN_NAMES if col not in BASE_COLS]
+            task_choices = TASKS
             with gr.Row():
                 tasks_select = gr.CheckboxGroup(
                     choices=task_choices,
@@ -99,12 +100,44 @@ with gr.Blocks() as block:
             refresh_button.click(fn=refresh_data, outputs=data_component)
 
         # table 2
-        with gr.TabItem("📝 About", elem_id="qa-tab-table2", id=2):
+        with gr.TabItem("📝 About: MMEB-V1", elem_id="qa-tab-table2", id=2):
             gr.Markdown(LEADERBOARD_INFO, elem_classes="markdown-text")
 
         # table 3
-        with gr.TabItem("🚀 Submit here! ", elem_id="submit-tab", id=3):
+        with gr.TabItem("🚀 Submit here for MMEB-V1! ", elem_id="submit-tab", id=3):
             with gr.Row():
                 gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
 
+        # table 4
+        with gr.TabItem("📊 NEW! MMEB-V2", elem_id="qa-tab-table1", id=1):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT_V2,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        lines=10,
+                    )
+            gr.Markdown(TABLE_INTRODUCTION_V2)
+            
+            df2 = get_df('results_v2.jsonl')
+
+            data_component_v2 = gr.components.Dataframe(
+                value=df2[COLUMN_NAMES_V2],
+                headers=COLUMN_NAMES_V2,
+                type="pandas",
+                datatype=DATA_TITLE_TYPE_V2,
+                interactive=False,
+                visible=True,
+                max_height=2400, 
+            )
+            
+            refresh_button2 = gr.Button("Refresh")
+            
+            def update_with_tasks(*args):
+                return update_table(*args)
+
+            refresh_button2.click(fn=lambda: refresh_data('results_v2.jsonl', COLUMN_NAMES_V2), outputs=data_component_v2)
+
+
 block.launch(share=True)

labels_v2.py

@@ -0,0 +1,39 @@
+TASKS_V2 = ['V-CLS', 'V-QA', 'V-RET', 'V-MRET']
+
+COLUMN_NAMES_V2 = [
+    "Rank", "Models", "Model Size(B)", "Data Source",
+    "V2-Overall",
+    "V-CLS", "V-QA", "V-RET", "V-MRET", "VisDoc"
+]
+
+BASE_COLS_V2 = [col for col in COLUMN_NAMES_V2 if col not in TASKS_V2]
+
+DATA_TITLE_TYPE_V2 = ['number', 'markdown', 'str', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number']
+
+TABLE_INTRODUCTION_V2 = """MMEB-V2"""
+
+CITATION_BUTTON_TEXT_V2 = r"""TBA"""
+
+SUBMIT_INTRODUCTION_V2 = """# Submit on MMEB-V2 Leaderboard Introduction
+
+## ⚠ Please note that you need to submit the JSON file with the following format:
+```json
+[
+    {
+        "Model": "<Model Name>",
+        <Optional>"URL": "<Model URL>",
+        "Model Size(B)": 1000,
+        "Data Source": Self-Reported,
+        "V2-Overall": 50.0,
+        "V-CLS": 50.0,
+        "V-QA": 50.0,
+        "V-RET": 50.0,
+        "V-MRET": 50.0, 
+        "VisDoc": 50.0
+    }, 
+]
+```
+You may refer to the Github page for instructions about evaluating your model. 
+Github link: https://github.com/TIGER-AI-Lab/VLM2Vec. \n
+Please send us an email at m7su@uwaterloo.ca, attaching the JSON file. We will review your submission and update the leaderboard accordingly.
+"""

results_v2.jsonl

@@ -0,0 +1 @@
+{"Models":"Model","Model Size(B)":null,"Data Source":null,"V2-Overall":0,"V-CLS":0,"V-QA":0,"V-RET":0,"V-MRET":0,"VisDoc": 0, "URL":null}

utils.py

@@ -12,23 +12,16 @@ HF_TOKEN = os.environ.get("HF_TOKEN")
 
 TASKS = ["Classification", "VQA", "Retrieval", "Grounding"]
 
-MODEL_INFO = [
+COLUMN_NAMES = [
     "Rank", "Models", "Model Size(B)", "Data Source",
     "Overall",
     "Classification", "VQA", "Retrieval", "Grounding"
 ]
 
-BASE_COLS = [col for col in MODEL_INFO if col not in TASKS]
+BASE_COLS = [col for col in COLUMN_NAMES if col not in TASKS]
 
 DATA_TITLE_TYPE = ['number', 'markdown', 'str', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
 
-SUBMISSION_NAME = "MMEB"
-SUBMISSION_URL = os.path.join("https://huggingface.co/spaces/TIGER-Lab/", SUBMISSION_NAME)
-FILE_NAME = "results.csv"
-CSV_DIR = "results.csv"
-
-COLUMN_NAMES = MODEL_INFO
-
 LEADERBOARD_INTRODUCTION = """
 # MMEB Leaderboard
 
@@ -45,7 +38,7 @@ Github link: https://github.com/TIGER-AI-Lab/VLM2Vec. \n
 Overview: https://tiger-ai-lab.github.io/VLM2Vec/. \n
 """
 
-TABLE_INTRODUCTION = """"""
+TABLE_INTRODUCTION = """MMEB-V1"""
 
 LEADERBOARD_INFO = """
 ## Dataset Summary
@@ -78,7 +71,7 @@ CITATION_BUTTON_TEXT = r"""@article{jiang2024vlm2vec,
   year={2024}
 }"""
 
-SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction
+SUBMIT_INTRODUCTION = """# Submit on MMEB-V1 Leaderboard Introduction
 
 ## ⚠ Please note that you need to submit the JSON file with the following format:
 ```json
@@ -126,20 +119,17 @@ def fetch_data(file: str) -> pd.DataFrame:
 
 def get_df(file="results.jsonl"):
     df = fetch_data(file)
-    print(df.columns)
-    print('URL' in df.columns)
-    print(df)
     df['Model Size(B)'] = df['Model Size(B)'].apply(process_model_size)
-    df = df.sort_values(by=['Overall'], ascending=False)
+    df = df.sort_values(by=[col for col in df.columns if 'Overall' in col], ascending=False)
     df = create_hyperlinked_names(df)
     df['Rank'] = range(1, len(df) + 1)
     return df
 
-def refresh_data():
-    df = get_df()
-    return df[COLUMN_NAMES]
+def refresh_data(file='results.jsonl', column_names=COLUMN_NAMES):
+    df = get_df(file)
+    return df[column_names]
 
-def search_and_filter_models(df, query, min_size, max_size):
+def search_and_filter_models(df, query, min_size, max_size, column_names=COLUMN_NAMES):
     filtered_df = df.copy()
     
     if query:
@@ -151,7 +141,7 @@ def search_and_filter_models(df, query, min_size, max_size):
     
     filtered_df = filtered_df[size_mask]
     
-    return filtered_df[COLUMN_NAMES]
+    return filtered_df[column_names]
 
 
 def search_models(df, query):
@@ -176,9 +166,9 @@ def process_model_size(size):
         return 'unknown'
 
 
-def filter_columns_by_tasks(df, selected_tasks=None):
+def filter_columns_by_tasks(df, selected_tasks=None, column_names=COLUMN_NAMES):
     if selected_tasks is None or len(selected_tasks) == 0:
-        return df[COLUMN_NAMES]
+        return df[column_names]
     
     base_columns = ['Models', 'Model Size(B)', 'Data Source', 'Overall']
     selected_columns = base_columns + selected_tasks
@@ -186,6 +176,5 @@ def filter_columns_by_tasks(df, selected_tasks=None):
     available_columns = [col for col in selected_columns if col in df.columns]
     return df[available_columns]
 
-def get_task_choices():
-    return TASKS
-
+def get_task_choices(tasks=TASKS):
+    return tasks