Add more model results, add single-image setting table

app.py

@@ -1,27 +1,35 @@
 import gradio as gr
-from utils import get_leaderboard_data, SUPER_GROUPS, MODEL_GROUPS
+from utils import DefaultDataLoader, CoreSingleDataLoader
 import os
 from constants import *
 
 # Get the directory of the current script
 current_dir = os.path.dirname(os.path.abspath(__file__))
 
-# Construct the path to the CSS file
-css_file = os.path.join(current_dir, "static", "css", "style.css")
+# Construct paths to CSS files
+base_css_file = os.path.join(current_dir, "static", "css", "style.css")
+default_css_file = os.path.join(current_dir, "static", "css", "default.css")
+core_single_css_file = os.path.join(current_dir, "static", "css", "core_single.css")
 
-# Read the CSS file
-with open(css_file, "r") as f:
-    css = f.read()
+# Read CSS files
+with open(base_css_file, "r") as f:
+    base_css = f.read()
+with open(default_css_file, "r") as f:
+    default_css = f.read()
+with open(core_single_css_file, "r") as f:
+    core_single_css = f.read()
 
-def update_leaderboard(selected_super_group, selected_model_group):
-    headers, data = get_leaderboard_data(selected_super_group, selected_model_group)
-    return gr.Dataframe(
-        value=data,
-        headers=headers,
-        datatype=["str"] + ["number"] * (len(headers) - 1),
-    )
+# Initialize data loaders
+default_loader = DefaultDataLoader()
+core_single_loader = CoreSingleDataLoader()
 
-with gr.Blocks(css=css) as block:
+with gr.Blocks() as block:
+    # Add a style element that we'll update
+    css_style = gr.HTML(
+        f"<style>{base_css}\n{default_css}</style>",
+        visible=False
+    )
+    
     gr.Markdown(
         LEADERBOARD_INTRODUCTION
     )
@@ -39,24 +47,36 @@ with gr.Blocks(css=css) as block:
                 TABLE_INTRODUCTION
             )
 
+            with gr.Row():
+                table_selector = gr.Radio(
+                    choices=["Default", "Core Single-image"],
+                    label="Select table to display",
+                    value="Default"
+                )
+
+            # Define different captions for each table
+            default_caption = "**Table 1: MEGA-Bench full results.** <br> The Core set contains $N_{\\text{core}} = 440$ tasks evaluated by rule-based metrics, and the Open-ended set contains $N_{\\text{open}} = 65$ tasks evaluated by a VLM judge (we use GPT-4o-0806). <br> $\\text{Overall} \\ = \\ \\frac{\\max(\\text{Core w/o CoT}, \\ \\text{Core w/ CoT}) \\ \\cdot \\ N_{\\text{core}} \\ + \\ \\text{Open-ended} \\ \\cdot \\ N_{\\text{open}}}{N_{\\text{core}} \\ + \\ N_{\\text{open}}}$"
+            core_single_image_caption = "**Table 2: MEGA-Bench Core Single-image results.** <br> This subset contains 273 single-image tasks from the Core set of the benchmark. For open-source models, we drop the image input in the 1-shot demonstration example so that the entire query contains a single image only. Compared to the default table, some models with only single-image support are added."
+
+            caption_component = gr.Markdown(
+                value=default_caption,
+                elem_classes="table-caption",
+                latex_delimiters=[{"left": "$", "right": "$", "display": False}],
+            )
+
             with gr.Row():
                 super_group_selector = gr.Radio(
-                    choices=list(SUPER_GROUPS.keys()),
+                    choices=list(default_loader.SUPER_GROUPS.keys()),
                     label="Select a dimension to display breakdown results. We use different column colors to distinguish the overall benchmark scores and breakdown results.",
-                    value=list(SUPER_GROUPS.keys())[0]
+                    value=list(default_loader.SUPER_GROUPS.keys())[0]
                 )
                 model_group_selector = gr.Radio(
-                    choices=list(MODEL_GROUPS.keys()),
+                    choices=list(default_loader.BASE_MODEL_GROUPS.keys()),
                     label="Select a model group",
                     value="All"
                 )
 
-            initial_headers, initial_data = get_leaderboard_data(list(SUPER_GROUPS.keys())[0], "All")
-            gr.Markdown(
-                "**Table 1: MEGA-Bench full results.** <br> The Core set contains $N_{\\text{core}} = 440$ tasks evaluated by rule-based metrics, and the Open-ended set contains $N_{\\text{open}} = 65$ tasks evaluated by a VLM judge (we use GPT-4o-0806). <br> $\\text{Overall} \\ = \\ \\frac{\\max(\\text{Core w/o CoT}, \\ \\text{Core w/ CoT}) \\ \\cdot \\ N_{\\text{core}} \\ + \\ \\text{Open-ended} \\ \\cdot \\ N_{\\text{open}}}{N_{\\text{core}} \\ + \\ N_{\\text{open}}}$",
-                elem_classes="table-caption", 
-                latex_delimiters=[ {"left": "$", "right": "$", "display": False }],
-            )
+            initial_headers, initial_data = default_loader.get_leaderboard_data(list(default_loader.SUPER_GROUPS.keys())[0], "All")
             data_component = gr.Dataframe(
                 value=initial_data,
                 headers=initial_headers,
@@ -65,10 +85,61 @@ with gr.Blocks(css=css) as block:
                 elem_classes="custom-dataframe",
                 max_height=1200,
             )
+
+            def update_table_and_caption(table_type, super_group, model_group):
+                if table_type == "Default":
+                    headers, data = default_loader.get_leaderboard_data(super_group, model_group)
+                    caption = default_caption
+                    current_css = f"{base_css}\n{default_css}"
+                else:  # Core Single-image
+                    headers, data = core_single_loader.get_leaderboard_data(super_group, model_group)
+                    caption = core_single_image_caption
+                    current_css = f"{base_css}\n{core_single_css}"
+                
+                return [
+                    gr.Dataframe(
+                        value=data,
+                        headers=headers,
+                        datatype=["str"] + ["number"] * (len(headers) - 1),
+                    ),
+                    caption,
+                    f"<style>{current_css}</style>"
+                ]
+
+            def update_selectors(table_type):
+                loader = default_loader if table_type == "Default" else core_single_loader
+                return [
+                    gr.Radio(choices=list(loader.SUPER_GROUPS.keys())),
+                    gr.Radio(choices=list(loader.MODEL_GROUPS.keys()))
+                ]
+
             refresh_button = gr.Button("Refresh")
-            refresh_button.click(fn=update_leaderboard, inputs=[super_group_selector, model_group_selector], outputs=[data_component])
-            super_group_selector.change(fn=update_leaderboard, inputs=[super_group_selector, model_group_selector], outputs=[data_component])
-            model_group_selector.change(fn=update_leaderboard, inputs=[super_group_selector, model_group_selector], outputs=[data_component])
+            
+            # Update click and change handlers to include caption updates
+            refresh_button.click(
+                fn=update_table_and_caption, 
+                inputs=[table_selector, super_group_selector, model_group_selector], 
+                outputs=[data_component, caption_component, css_style]
+            )
+            super_group_selector.change(
+                fn=update_table_and_caption, 
+                inputs=[table_selector, super_group_selector, model_group_selector], 
+                outputs=[data_component, caption_component, css_style]
+            )
+            model_group_selector.change(
+                fn=update_table_and_caption, 
+                inputs=[table_selector, super_group_selector, model_group_selector], 
+                outputs=[data_component, caption_component, css_style]
+            )
+            table_selector.change(
+                fn=update_selectors,
+                inputs=[table_selector],
+                outputs=[super_group_selector, model_group_selector]
+            ).then(
+                fn=update_table_and_caption,
+                inputs=[table_selector, super_group_selector, model_group_selector],
+                outputs=[data_component, caption_component, css_style]
+            )
 
         with gr.TabItem("📝 Data Information", elem_id="qa-tab-table2", id=2):
             gr.Markdown(DATA_INFO, elem_classes="markdown-text")

static/css/core_single.css

@@ -0,0 +1,57 @@
+.custom-dataframe thead th:nth-child(-n+2),
+.custom-dataframe tbody td:nth-child(-n+2) {
+    background-color: var(--global-column-background) !important;
+}
+
+.custom-dataframe thead th:nth-child(n+3),
+.custom-dataframe tbody td:nth-child(n+3) {
+    background-color: var(--dimension-column-background) !important;
+}
+
+.custom-dataframe tbody tr:nth-child(even) td:nth-child(-n+2) {
+    background-color: var(--row-even-global) !important;
+}
+
+.custom-dataframe tbody tr:nth-child(even) td:nth-child(n+3) {
+    background-color: var(--row-even-dimension) !important;
+}
+
+/* Dark mode styles */
+@media (prefers-color-scheme: dark) {
+    .custom-dataframe {
+        color: var(--text-color) !important;
+        background-color: var(--background-color) !important;
+    }
+
+    .custom-dataframe thead th {
+        background-color: var(--header-background) !important;
+        color: var(--text-color) !important;
+    }
+
+    .custom-dataframe tbody td {
+        background-color: var(--background-color) !important;
+        color: var(--text-color) !important;
+    }
+
+    .custom-dataframe thead th:nth-child(-n+2),
+    .custom-dataframe tbody td:nth-child(-n+2) {
+        background-color: var(--global-column-background) !important;
+    }
+
+    .custom-dataframe thead th:nth-child(n+3),
+    .custom-dataframe tbody td:nth-child(n+3) {
+        background-color: var(--dimension-column-background) !important;
+    }
+
+    .custom-dataframe tbody tr:nth-child(even) td:nth-child(-n+2) {
+        background-color: var(--row-even-global) !important;
+    }
+
+    .custom-dataframe tbody tr:nth-child(even) td:nth-child(n+3) {
+        background-color: var(--row-even-dimension) !important;
+    }
+
+    .custom-dataframe tbody tr:hover td {
+        background-color: var(--hover-background) !important;
+    }
+}

static/css/default.css

@@ -0,0 +1,57 @@
+.custom-dataframe thead th:nth-child(-n+5),
+.custom-dataframe tbody td:nth-child(-n+5) {
+    background-color: var(--global-column-background) !important;
+}
+
+.custom-dataframe thead th:nth-child(n+6),
+.custom-dataframe tbody td:nth-child(n+6) {
+    background-color: var(--dimension-column-background) !important;
+}
+
+.custom-dataframe tbody tr:nth-child(even) td:nth-child(-n+5) {
+    background-color: var(--row-even-global) !important;
+}
+
+.custom-dataframe tbody tr:nth-child(even) td:nth-child(n+6) {
+    background-color: var(--row-even-dimension) !important;
+}
+
+/* Dark mode styles */
+@media (prefers-color-scheme: dark) {
+    .custom-dataframe {
+        color: var(--text-color) !important;
+        background-color: var(--background-color) !important;
+    }
+
+    .custom-dataframe thead th {
+        background-color: var(--header-background) !important;
+        color: var(--text-color) !important;
+    }
+
+    .custom-dataframe tbody td {
+        background-color: var(--background-color) !important;
+        color: var(--text-color) !important;
+    }
+
+    .custom-dataframe thead th:nth-child(-n+5),
+    .custom-dataframe tbody td:nth-child(-n+5) {
+        background-color: var(--global-column-background) !important;
+    }
+
+    .custom-dataframe thead th:nth-child(n+6),
+    .custom-dataframe tbody td:nth-child(n+6) {
+        background-color: var(--dimension-column-background) !important;
+    }
+
+    .custom-dataframe tbody tr:nth-child(even) td:nth-child(-n+5) {
+        background-color: var(--row-even-global) !important;
+    }
+
+    .custom-dataframe tbody tr:nth-child(even) td:nth-child(n+6) {
+        background-color: var(--row-even-dimension) !important;
+    }
+
+    .custom-dataframe tbody tr:hover td {
+        background-color: var(--hover-background) !important;
+    }
+}

static/css/style.css

@@ -40,64 +40,6 @@
     color: var(--text-color) !important;
 }
 
-.custom-dataframe thead th:nth-child(-n+5),
-.custom-dataframe tbody td:nth-child(-n+5) {
-    background-color: var(--global-column-background) !important;
-}
-
-.custom-dataframe thead th:nth-child(n+6),
-.custom-dataframe tbody td:nth-child(n+6) {
-    background-color: var(--dimension-column-background) !important;
-}
-
-.custom-dataframe tbody tr:nth-child(even) td:nth-child(-n+5) {
-    background-color: var(--row-even-global) !important;
-}
-
-.custom-dataframe tbody tr:nth-child(even) td:nth-child(n+6) {
-    background-color: var(--row-even-dimension) !important;
-}
-
-/* Dark mode styles */
-@media (prefers-color-scheme: dark) {
-    .custom-dataframe {
-        color: var(--text-color) !important;
-        background-color: var(--background-color) !important;
-    }
-
-    .custom-dataframe thead th {
-        background-color: var(--header-background) !important;
-        color: var(--text-color) !important;
-    }
-
-    .custom-dataframe tbody td {
-        background-color: var(--background-color) !important;
-        color: var(--text-color) !important;
-    }
-
-    .custom-dataframe thead th:nth-child(-n+5),
-    .custom-dataframe tbody td:nth-child(-n+5) {
-        background-color: var(--global-column-background) !important;
-    }
-
-    .custom-dataframe thead th:nth-child(n+6),
-    .custom-dataframe tbody td:nth-child(n+6) {
-        background-color: var(--dimension-column-background) !important;
-    }
-
-    .custom-dataframe tbody tr:nth-child(even) td:nth-child(-n+5) {
-        background-color: var(--row-even-global) !important;
-    }
-
-    .custom-dataframe tbody tr:nth-child(even) td:nth-child(n+6) {
-        background-color: var(--row-even-dimension) !important;
-    }
-
-    .custom-dataframe tbody tr:hover td {
-        background-color: var(--hover-background) !important;
-    }
-}
-
 .table-caption {
     text-align: center;
     margin-top: 10px;

static/eval_results/Core_SI/all_summary.json

@@ -0,0 +1,227 @@
+{
+    "Aquila_VL_2B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.20770364903712493,
+        "micro_mean_score": 0.20333142638522636,
+        "missing_tasks": []
+    },
+    "Aria": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.3178882776147889,
+        "micro_mean_score": 0.3101511832828904,
+        "missing_tasks": []
+    },
+    "Claude_3.5": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4116,
+        "macro_mean_score": 0.520276385877485,
+        "micro_mean_score": 0.520276385877485
+    },
+    "Claude_3.5_new": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4116,
+        "macro_mean_score": 0.5462752278980763,
+        "micro_mean_score": 0.5462752278980763
+    },
+    "GPT_4o": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4116,
+        "macro_mean_score": 0.5529953662872719,
+        "micro_mean_score": 0.5529953662872719
+    },
+    "GPT_4o_mini": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4116,
+        "macro_mean_score": 0.44285970964797233,
+        "micro_mean_score": 0.44285970964797233
+    },
+    "Gemini_1.5_flash_002": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4116,
+        "macro_mean_score": 0.42188460865574384,
+        "micro_mean_score": 0.42188460865574384
+    },
+    "Gemini_1.5_pro_002": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4116,
+        "macro_mean_score": 0.4914311038229404,
+        "micro_mean_score": 0.4914311038229404
+    },
+    "Idefics3": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.08941182847569326,
+        "micro_mean_score": 0.08779475233900695,
+        "missing_tasks": []
+    },
+    "InternVL2_2B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.12069001041308772,
+        "micro_mean_score": 0.11842605219090299,
+        "missing_tasks": []
+    },
+    "InternVL2_76B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.3998616568018755,
+        "micro_mean_score": 0.39149064302628933,
+        "missing_tasks": []
+    },
+    "InternVL2_8B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.27650612401825575,
+        "micro_mean_score": 0.27119471729837735,
+        "missing_tasks": []
+    },
+    "Llama_3_2_11B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.20789144960796493,
+        "micro_mean_score": 0.20163641703273802,
+        "missing_tasks": []
+    },
+    "MiniCPM_v2.6": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.23230765810722817,
+        "micro_mean_score": 0.22684118052665975,
+        "missing_tasks": []
+    },
+    "Molmo_72B": {
+        "num_eval_tasks": 270,
+        "num_eval_samples": 4073,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4331,
+        "macro_mean_score": 0.36480000609384927,
+        "micro_mean_score": 0.36205779758110807,
+        "missing_tasks": [
+            "table_understanding",
+            "MMSoc_Misinformation_PolitiFact",
+            "planning_screenshot_termes"
+        ]
+    },
+    "Molmo_7B_D": {
+        "num_eval_tasks": 272,
+        "num_eval_samples": 4102,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4362,
+        "macro_mean_score": 0.2098088446992518,
+        "micro_mean_score": 0.20550929661464645,
+        "missing_tasks": [
+            "MMSoc_Misinformation_PolitiFact"
+        ]
+    },
+    "NVLM": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.32989872890926025,
+        "micro_mean_score": 0.32315683713111915,
+        "missing_tasks": []
+    },
+    "POINTS_7B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.25511317681632334,
+        "micro_mean_score": 0.24927711632415062,
+        "missing_tasks": []
+    },
+    "Phi-3.5-vision": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.2561274958722834,
+        "micro_mean_score": 0.2504214576875906,
+        "missing_tasks": []
+    },
+    "Pixtral_12B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.3436942439614412,
+        "micro_mean_score": 0.3373564384613738,
+        "missing_tasks": []
+    },
+    "Qwen2_VL_2B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.22787906973244856,
+        "micro_mean_score": 0.2234748515064842,
+        "missing_tasks": []
+    },
+    "Qwen2_VL_72B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.4730536307784527,
+        "micro_mean_score": 0.4659830915476831,
+        "missing_tasks": []
+    },
+    "Qwen2_VL_7B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.3538656561495699,
+        "micro_mean_score": 0.34581250459157137,
+        "missing_tasks": []
+    },
+    "llava_onevision_72B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.312618242621264,
+        "micro_mean_score": 0.3098623876487132,
+        "missing_tasks": []
+    },
+    "llava_onevision_7B": {
+        "num_eval_tasks": 273,
+        "num_eval_samples": 4116,
+        "num_not_eval_samples": 0,
+        "num_total_samples": 4377,
+        "macro_mean_score": 0.23683339637631812,
+        "micro_mean_score": 0.23283041278687175,
+        "missing_tasks": []
+    }
+}

static/eval_results/{all_summary.json → Default/all_summary.json}

@@ -5,16 +5,16 @@
             "num_eval_samples": 6539,
             "num_not_eval_samples": 0,
             "num_total_samples": 6961,
-            "macro_mean_score": 0.5203470034386184,
-            "micro_mean_score": 0.514305381949725
+            "macro_mean_score": 0.5203440930873326,
+            "micro_mean_score": 0.514302640282204
         },
         "core_cot": {
             "num_eval_tasks": 440,
             "num_eval_samples": 6539,
             "num_not_eval_samples": 0,
             "num_total_samples": 6961,
-            "macro_mean_score": 0.5265059698578094,
-            "micro_mean_score": 0.5236365938368621
+            "macro_mean_score": 0.5265030595065238,
+            "micro_mean_score": 0.5236338521693411
         },
         "open": {
             "num_eval_tasks": 65,
@@ -23,7 +23,7 @@
             "macro_mean_score": 0.6478225794744895,
             "micro_mean_score": 0.665391229578676
         },
-        "overall_score": 0.542120979016392
+        "overall_score": 0.5421184432647768
     },
     "Gemini_1.5_pro_002": {
         "core_noncot": {
@@ -117,8 +117,8 @@
             "num_eval_samples": 6539,
             "num_not_eval_samples": 0,
             "num_total_samples": 6961,
-            "macro_mean_score": 0.525918992480593,
-            "micro_mean_score": 0.5230784020211157
+            "macro_mean_score": 0.5259191914020757,
+            "micro_mean_score": 0.5230785894131227
         },
         "open": {
             "num_eval_tasks": 65,
@@ -127,7 +127,7 @@
             "macro_mean_score": 0.6563419761104125,
             "micro_mean_score": 0.6724419604471196
         },
-        "overall_score": 0.5427061091854214
+        "overall_score": 0.5427062825031487
     },
     "GPT_4o_mini": {
         "core_noncot": {
@@ -492,5 +492,83 @@
             "micro_mean_score": 0.3947549441100602
         },
         "overall_score": 0.25566537510391796
+    },
+    "InternVL2_2B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.09089701489596874,
+            "micro_mean_score": 0.09036328295381871
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.13141974398938763,
+            "micro_mean_score": 0.13063500716262516
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.23864417043743646,
+            "micro_mean_score": 0.24901117798796224
+        },
+        "overall_score": 0.14522090778963154
+    },
+    "Qwen2_VL_2B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.16448220309703876,
+            "micro_mean_score": 0.1610710186451323
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.20877163406364055,
+            "micro_mean_score": 0.20561526268932287
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.3154302566225611,
+            "micro_mean_score": 0.33856405846947557
+        },
+        "overall_score": 0.22249997162072932
+    },
+    "Aquila_VL_2B": {
+        "core_noncot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.16317824309838627,
+            "micro_mean_score": 0.16198837245148487
+        },
+        "core_cot": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "num_not_eval_samples": 0,
+            "num_total_samples": 6961,
+            "macro_mean_score": 0.159970161379836,
+            "micro_mean_score": 0.15844711671722148
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "num_total_samples": 1224,
+            "macro_mean_score": 0.24567572098570653,
+            "micro_mean_score": 0.2704213241616509
+        },
+        "overall_score": 0.17379673035120966
     }
 }

utils.py

@@ -1,21 +1,8 @@
 import pandas as pd
-import gradio as gr
-import csv
 import json
-import os
-import shutil
-from huggingface_hub import Repository
-import numpy as np
+from typing import Dict, Any, Tuple
 
-# Load the JSON data
-with open("./static/eval_results/all_model_keywords_stats.json", "r") as f:
-    MODEL_DATA = json.load(f)
-
-with open("./static/eval_results/all_summary.json", "r") as f:
-    SUMMARY_DATA = json.load(f)
-
-
-# Define model name mapping
+# Keep all the constant mappings outside the class
 MODEL_NAME_MAP = {
     "Claude_3.5_new": "Claude-3.5-Sonnet (1022)",
     "GPT_4o": "GPT-4o (0513)",
@@ -36,9 +23,14 @@ MODEL_NAME_MAP = {
     "Phi-3.5-vision": "Phi-3.5-Vision",
     "MiniCPM_v2.6": "MiniCPM-V2.6",
     "Idefics3": "Idefics3-8B-Llama3",
+    "Aquila_VL_2B": "Aquila-VL-2B-llava-qwen",
+    "POINTS_7B": "POINTS-Qwen2.5-7B",
+    "Qwen2_VL_2B": "Qwen2-VL-2B",
+    "InternVL2_2B": "InternVL2-2B",
+    "Molmo_7B_D": "Molmo-7B-D-0924",
+    "Molmo_72B": "Molmo-72B-0924",
 }
 
-# Custom name mapping for dimensions and keywords
 DIMENSION_NAME_MAP = {
     "skills": "Skills",
     "input_format": "Input Format",
@@ -91,59 +83,157 @@ KEYWORD_NAME_MAP = {
     "video": "Video",
 }
 
-# Extract super groups (dimensions) and their keywords
-SUPER_GROUPS = {DIMENSION_NAME_MAP[dim]: [KEYWORD_NAME_MAP.get(k, k) for k in MODEL_DATA[next(iter(MODEL_DATA))][dim].keys()] 
-                for dim in MODEL_DATA[next(iter(MODEL_DATA))]}
+class BaseDataLoader:
+    # Define the base MODEL_GROUPS structure
+    BASE_MODEL_GROUPS = {
+        "All": list(MODEL_NAME_MAP.keys()),
+        "Flagship Models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM'],
+        "Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
+        "Proprietary Flagship models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002'],
+        "Proprietary Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini'],
+        "Open-source Flagship Models": ['Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM'],
+        "Open-source Efficiency Models": ['Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
+    }
+
+    def __init__(self):
+        self.MODEL_DATA = self._load_model_data()
+        self.SUMMARY_DATA = self._load_summary_data()
+        self.SUPER_GROUPS = self._initialize_super_groups()
+        self.MODEL_GROUPS = self._initialize_model_groups()
+
+    def _initialize_super_groups(self):
+        # Define the desired order of super groups
+        
+        groups = {DIMENSION_NAME_MAP[dim]: [KEYWORD_NAME_MAP.get(k, k) for k in self.MODEL_DATA[next(iter(self.MODEL_DATA))][dim].keys()] 
+                 for dim in self.MODEL_DATA[next(iter(self.MODEL_DATA))]}
+        
+        order = ["Skills", "Application", "Output Format", "Input Format", "Visual Input Number"]
+        # Sort the dictionary based on the predefined order
+        return {k: groups[k] for k in order if k in groups}
+
+    def _initialize_model_groups(self) -> Dict[str, list]:
+        # Get the list of available models from the loaded data
+        available_models = set(self.MODEL_DATA.keys())
+        
+        # Create filtered groups based on available models
+        filtered_groups = {}
+        for group_name, models in self.BASE_MODEL_GROUPS.items():
+            if group_name == "All":
+                filtered_groups[group_name] = sorted(list(available_models))
+            else:
+                filtered_models = [model for model in models if model in available_models]
+                if filtered_models:  # Only include group if it has models
+                    filtered_groups[group_name] = filtered_models
+        
+        return filtered_groups
+
+    def _load_model_data(self) -> Dict[str, Any]:
+        raise NotImplementedError("Subclasses must implement _load_model_data")
+
+    def _load_summary_data(self) -> Dict[str, Any]:
+        raise NotImplementedError("Subclasses must implement _load_summary_data")
+
+    def get_df(self, selected_super_group: str, selected_model_group: str) -> pd.DataFrame:
+        raise NotImplementedError("Subclasses must implement get_df")
+
+    def get_leaderboard_data(self, selected_super_group: str, selected_model_group: str) -> Tuple[list, list]:
+        raise NotImplementedError("Subclasses must implement get_leaderboard_data")
+
+
+class DefaultDataLoader(BaseDataLoader):
+    def __init__(self):
+        super().__init__()
+
+    def _load_model_data(self) -> Dict[str, Any]:
+        with open("./static/eval_results/Default/all_model_keywords_stats.json", "r") as f:
+            return json.load(f)
 
+    def _load_summary_data(self) -> Dict[str, Any]:
+        with open("./static/eval_results/Default/all_summary.json", "r") as f:
+            return json.load(f)
+
+    def get_df(self, selected_super_group: str, selected_model_group: str) -> pd.DataFrame:
+        original_dimension = get_original_dimension(selected_super_group)
+        data = []
+        for model in self.MODEL_GROUPS[selected_model_group]:
+            model_data = self.MODEL_DATA[model]
+            summary = self.SUMMARY_DATA[model]
+            core_noncot_score = summary["core_noncot"]["macro_mean_score"]
+            core_cot_score = summary["core_cot"]["macro_mean_score"]
+            row = {
+                "Models": get_display_model_name(model),
+                "Overall": round(summary["overall_score"] * 100, 2),
+                "Core(w/o CoT)": round(core_noncot_score * 100, 2),
+                "Core(w/ CoT)": round(core_cot_score * 100, 2),
+                "Open-ended": round(summary["open"]["macro_mean_score"] * 100, 2)
+            }
+            for keyword in self.SUPER_GROUPS[selected_super_group]:
+                original_keyword = get_original_keyword(keyword)
+                if original_dimension in model_data and original_keyword in model_data[original_dimension]:
+                    row[keyword] = round(model_data[original_dimension][original_keyword]["average_score"] * 100, 2)
+                else:
+                    row[keyword] = None
+            data.append(row)
+        
+        df = pd.DataFrame(data)
+        df = df.sort_values(by="Overall", ascending=False)
+        return df
+
+    def get_leaderboard_data(self, selected_super_group: str, selected_model_group: str) -> Tuple[list, list]:
+        df = self.get_df(selected_super_group, selected_model_group)
+        headers = ["Models", "Overall", "Core(w/o CoT)", "Core(w/ CoT)", "Open-ended"] + self.SUPER_GROUPS[selected_super_group]
+        data = df[headers].values.tolist()
+        return headers, data
+
+
+class CoreSingleDataLoader(BaseDataLoader):
+    def __init__(self):
+        super().__init__()
+
+    def _load_model_data(self) -> Dict[str, Any]:
+        with open("./static/eval_results/Core_SI/all_model_keywords_stats.json", "r") as f:
+            return json.load(f)
+
+    def _load_summary_data(self) -> Dict[str, Any]:
+        with open("./static/eval_results/Core_SI/all_summary.json", "r") as f:
+            return json.load(f)
+
+    def get_df(self, selected_super_group: str, selected_model_group: str) -> pd.DataFrame:
+        original_dimension = get_original_dimension(selected_super_group)
+        data = []
+        for model in self.MODEL_GROUPS[selected_model_group]:
+            model_data = self.MODEL_DATA[model]
+            summary = self.SUMMARY_DATA[model]
+            core_si_score = summary["macro_mean_score"]
+            row = {
+                "Models": get_display_model_name(model),
+                "Core SI": round(core_si_score * 100, 2),
+            }
+            for keyword in self.SUPER_GROUPS[selected_super_group]:
+                original_keyword = get_original_keyword(keyword)
+                if original_dimension in model_data and original_keyword in model_data[original_dimension]:
+                    row[keyword] = round(model_data[original_dimension][original_keyword]["average_score"] * 100, 2)
+                else:
+                    row[keyword] = None
+            data.append(row)
+        
+        df = pd.DataFrame(data)
+        df = df.sort_values(by="Core SI", ascending=False)
+        return df
+
+    def get_leaderboard_data(self, selected_super_group: str, selected_model_group: str) -> Tuple[list, list]:
+        df = self.get_df(selected_super_group, selected_model_group)
+        headers = ["Models", "Core SI"] + self.SUPER_GROUPS[selected_super_group]
+        data = df[headers].values.tolist()
+        return headers, data
+
+
+# Keep your helper functions
 def get_original_dimension(mapped_dimension):
     return next(k for k, v in DIMENSION_NAME_MAP.items() if v == mapped_dimension)
 
 def get_original_keyword(mapped_keyword):
     return next((k for k, v in KEYWORD_NAME_MAP.items() if v == mapped_keyword), mapped_keyword)
 
-# Define model groups
-MODEL_GROUPS = {
-    "All": list(MODEL_DATA.keys()),
-    "Flagship Models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM'],
-    "Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
-    "Proprietary Flagship models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002'],
-    "Proprietary Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini'],
-    "Open-source Flagship Models": ['Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM'],
-    "Open-source Efficiency Models": ['Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3'],
-}
-
 def get_display_model_name(model_name):
     return MODEL_NAME_MAP.get(model_name, model_name)
-
-def get_df(selected_super_group, selected_model_group):
-    original_dimension = get_original_dimension(selected_super_group)
-    data = []
-    for model in MODEL_GROUPS[selected_model_group]:
-        model_data = MODEL_DATA[model]
-        summary = SUMMARY_DATA[model]
-        core_noncot_score = summary["core_noncot"]["macro_mean_score"]
-        core_cot_score = summary["core_cot"]["macro_mean_score"]
-        row = {
-            "Models": get_display_model_name(model),  # Use the mapped name
-            "Overall": round(summary["overall_score"] * 100, 2),
-            "Core(w/o CoT)": round(core_noncot_score * 100, 2),
-            "Core(w/ CoT)": round(core_cot_score * 100, 2),
-            "Open-ended": round(summary["open"]["macro_mean_score"] * 100, 2)
-        }
-        for keyword in SUPER_GROUPS[selected_super_group]:
-            original_keyword = get_original_keyword(keyword)
-            if original_dimension in model_data and original_keyword in model_data[original_dimension]:
-                row[keyword] = round(model_data[original_dimension][original_keyword]["average_score"] * 100, 2)
-            else:
-                row[keyword] = None
-        data.append(row)
-    
-    df = pd.DataFrame(data)
-    df = df.sort_values(by="Overall", ascending=False)
-    return df
-
-def get_leaderboard_data(selected_super_group, selected_model_group):
-    df = get_df(selected_super_group, selected_model_group)
-    headers = ["Models", "Overall", "Core(w/o CoT)", "Core(w/ CoT)", "Open-ended"] + SUPER_GROUPS[selected_super_group]
-    data = df[headers].values.tolist()
-    return headers, data