init commit

.gitignore

@@ -0,0 +1,13 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

app.py

@@ -1,204 +1,110 @@
-import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
+from utils import *
+
+global data_component
+
+def update_table(query, min_size, max_size, selected_subjects=None):
+    df = get_df()
+    filtered_df = search_and_filter_models(df, query, min_size, max_size)
+    if selected_subjects and len(selected_subjects) > 0:
+        base_columns = ['Models', 'Model Size(B)', 'Data Source', 'Overall']
+        selected_columns = base_columns + selected_subjects
+        filtered_df = filtered_df[selected_columns]
+    return filtered_df
+
+with gr.Blocks() as block:
+    gr.Markdown(LEADERBOARD_INTRODUCTION)
+    
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+        # Table 1
+        with gr.TabItem("📊 MMLU-Pro", elem_id="qa-tab-table1", id=1):
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        lines=10,
                     )
+            gr.Markdown(TABLE_INTRODUCTION)
 
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+            with gr.Row():
+                search_bar = gr.Textbox(
+                    placeholder="Search models...",
+                    show_label=False,
+                    elem_id="search-bar"
+                )
+            
+            df = get_df()
+            min_size, max_size = get_size_range(df)
 
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
+            with gr.Row():
+                min_size_slider = gr.Slider(
+                    minimum=min_size,
+                    maximum=max_size,
+                    value=min_size,
+                    step=0.1,
+                    label="Minimum number of parameters (B)",
+                )
+                max_size_slider = gr.Slider(
+                    minimum=min_size,
+                    maximum=max_size,
+                    value=max_size,
+                    step=0.1,
+                    label="Maximum number of parameters (B)",
+                )
+
+            subject_choices = [col for col in COLUMN_NAMES if col not in ['Models', 'Model Size(B)', 'Data Source', 'Overall', 'IND', 'OOD']]
+            with gr.Row():
+                subjects_select = gr.CheckboxGroup(
+                    choices=subject_choices,
+                    value=subject_choices,
+                    label="Select Subjects to Display",
+                    elem_id="subjects-select"
+                )
+
+            data_component = gr.components.Dataframe(
+                value=df[COLUMN_NAMES],
+                headers=COLUMN_NAMES,
+                type="pandas",
+                datatype=DATA_TITLE_TYPE,
+                interactive=False,
+                visible=True,
             )
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
+            
+            refresh_button = gr.Button("Refresh")
+            
+            def update_with_subjects(*args):
+                return update_table(*args)
+
+            search_bar.change(
+                fn=update_with_subjects, 
+                inputs=[search_bar, min_size_slider, max_size_slider, subjects_select], 
+                outputs=data_component
+            )
+            min_size_slider.change(
+                fn=update_with_subjects, 
+                inputs=[search_bar, min_size_slider, max_size_slider, subjects_select], 
+                outputs=data_component
+            )
+            max_size_slider.change(
+                fn=update_with_subjects, 
+                inputs=[search_bar, min_size_slider, max_size_slider, subjects_select], 
+                outputs=data_component
+            )
+            subjects_select.change(
+                fn=update_with_subjects, 
+                inputs=[search_bar, min_size_slider, max_size_slider, subjects_select], 
+                outputs=data_component
             )
+            refresh_button.click(fn=refresh_data, outputs=data_component)
+
+        # table 2
+        with gr.TabItem("📝 About", elem_id="qa-tab-table2", id=2):
+            gr.Markdown(LEADERBOARD_INFO, elem_classes="markdown-text")
+
+        # table 3
+        with gr.TabItem("🚀 Submit here! ", elem_id="submit-tab", id=3):
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+block.launch(share=True)

index.html

@@ -1,19 +0,0 @@
-<!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
-</html>

results.csv

@@ -0,0 +1,15 @@
+Models,Model Size(B),Data Source,Overall,IND,OOD,Classification,VQA,Retrieval,Grounding
+CLIP,unk,unk,37.8,37.1,38.7,42.8,9.1,53.0,51.8
+BLIP2,unk,unk,25.2,25.3,25.1,27.0,4.2,33.9,47.0
+SigLIP,unk,unk,34.8,32.3,38.0,40.3,8.4,31.6,59.5
+OpenCLIP,unk,unk,39.7,39.3,40.2,47.8,10.9,52.3,53.3
+UniIR (BLIP_FF),unk,unk,42.8,44.7,40.4,42.1,15.0,60.1,62.2
+UniIR (CLIP_SF),unk,unk,44.7,47.1,41.7,44.3,16.2,61.8,65.3
+E5-V,unk,unk,13.3,14.9,11.5,21.8,4.9,11.5,19.0
+Magiclens,unk,unk,27.8,31.0,23.7,38.8,8.3,35.4,26.0
+CLIP-FFT,unk,TIGER-Lab,45.4,47.6,42.8,55.2,19.7,53.2,62.2
+OpenCLIP-FFT,unk,unk,47.2,50.5,43.1,56.0,21.9,55.4,64.1
+VLM2Vec (Phi-3.5-V-FFT),unk,TIGER-Lab,55.9,62.8,47.4,52.8,50.3,57.8,72.3
+VLM2Vec (Phi-3.5-V-LoRA),unk,TIGER-Lab,60.1,66.5,52.0,54.8,54.9,62.3,79.5
+VLM2Vec (LLaVA-1.6-LoRA-LowRes),unk,TIGER-Lab,55.0,61.0,47.5,54.7,50.3,56.2,64.0
+VLM2Vec (LLaVA-1.6-LoRA-HighRes),unk,TIGER-Lab,62.9,67.5,57.1,61.2,49.9,67.4,86.1

src/about.py

@@ -21,7 +21,7 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">MMEB Leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """

src/display/css_html_js.py

@@ -48,6 +48,7 @@ custom_css = """
 
 .tab-buttons button {
     font-size: 20px;
+    height: 1500px;
 }
 
 #scale-logo {

style.css

@@ -1,28 +0,0 @@
-body {
-	padding: 2rem;
-	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
-}
-
-h1 {
-	font-size: 16px;
-	margin-top: 0;
-}
-
-p {
-	color: rgb(107, 114, 128);
-	font-size: 15px;
-	margin-bottom: 10px;
-	margin-top: 5px;
-}
-
-.card {
-	max-width: 620px;
-	margin: 0 auto;
-	padding: 16px;
-	border: 1px solid lightgray;
-	border-radius: 16px;
-}
-
-.card p:last-child {
-	margin-bottom: 0;
-}

utils.py

@@ -0,0 +1,183 @@
+import pandas as pd
+import gradio as gr
+import csv
+import json
+import os
+import shutil
+from huggingface_hub import Repository
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+
+SUBJECTS = ["Classification", "VQA", "Retrieval", "Grounding"]
+
+MODEL_INFO = [
+    "Models", "Model Size(B)", "Data Source",
+    "Overall", "IND", "OOD", 
+    "Classification", "VQA", "Retrieval", "Grounding"
+]
+
+DATA_TITLE_TYPE = ['markdown', 'str', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number', 'number']
+
+# TODO: submission process not implemented yet
+SUBMISSION_NAME = ""
+SUBMISSION_URL = ""
+CSV_DIR = "results.csv" # TODO: Temporary file, to be updated with the actual file
+
+COLUMN_NAMES = MODEL_INFO
+
+LEADERBOARD_INTRODUCTION = """# MMEB Leaderboard
+
+## Introduction
+We introduce MMEB, a benchmark for multimodal evaluation of models. The benchmark consists of four tasks: Classification, VQA, Retrieval, and Grounding. Models are evaluated based on 36 datasets. 
+
+
+"""
+
+TABLE_INTRODUCTION = """"""
+
+LEADERBOARD_INFO = """
+## Dataset Summary
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = """"""
+
+SUBMIT_INTRODUCTION = """# Submit on MMEB Leaderboard Introduction
+
+## ⚠ Please note that you need to submit the JSON file with the following format:
+```json
+[
+    {
+        "question_id": 123,
+        "question": "abc",
+        "options": ["abc", "xyz", ...], 
+        "answer": "ABC",
+        "answer_index": 1,
+        "category": "abc,
+        "pred": "B",
+        "model_outputs": ""
+    }, ...
+]
+```
+...
+"""
+
+def get_df():
+    # TODO: Update this after the hf dataset has been created!
+    # repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL, use_auth_token=HF_TOKEN)
+    # repo.git_pull()
+    df = pd.read_csv(CSV_DIR)
+    df['Model Size(B)'] = df['Model Size(B)'].apply(process_model_size)
+    df = df.sort_values(by=['Overall'], ascending=False)
+    return df
+
+
+def add_new_eval(
+    input_file,
+):
+    if input_file is None:
+        return "Error! Empty file!"
+
+    upload_data = json.loads(input_file)
+    print("upload_data:\n", upload_data)
+    data_row = [f'{upload_data["Model"]}', upload_data['Overall']]
+    for subject in SUBJECTS:
+        data_row += [upload_data[subject]]
+    print("data_row:\n", data_row)
+    submission_repo = Repository(local_dir=SUBMISSION_NAME, clone_from=SUBMISSION_URL,
+                                 use_auth_token=HF_TOKEN, repo_type="dataset")
+    submission_repo.git_pull()
+
+    already_submitted = []
+    with open(CSV_DIR, mode='r') as file:
+        reader = csv.reader(file, delimiter=',')
+        for row in reader:
+            already_submitted.append(row[0])
+
+    if data_row[0] not in already_submitted:
+        with open(CSV_DIR, mode='a', newline='') as file:
+            writer = csv.writer(file)
+            writer.writerow(data_row)
+        
+        submission_repo.push_to_hub()
+        print('Submission Successful')
+    else:
+        print('The entry already exists')
+
+def refresh_data():
+    df = get_df()
+    return df[COLUMN_NAMES]
+
+
+def search_and_filter_models(df, query, min_size, max_size):
+    filtered_df = df.copy()
+    
+    if query:
+        filtered_df = filtered_df[filtered_df['Models'].str.contains(query, case=False, na=False)]
+
+    size_mask = filtered_df['Model Size(B)'].apply(lambda x: 
+        (min_size <= 1000.0 <= max_size) if x == 'unknown' 
+        else (min_size <= x <= max_size))
+    
+    filtered_df = filtered_df[size_mask]
+    
+    return filtered_df[COLUMN_NAMES]
+
+
+# def search_and_filter_models(df, query, min_size, max_size):
+#     filtered_df = df.copy()
+
+#     if query:
+#         filtered_df = filtered_df[filtered_df['Models'].str.contains(query, case=False, na=False)]
+    
+#     def size_filter(x):
+#         if isinstance(x, (int, float)):
+#             return min_size <= x <= max_size
+#         return True 
+    
+#     filtered_df = filtered_df[filtered_df['Model Size(B)'].apply(size_filter)]
+    
+#     return filtered_df[COLUMN_NAMES]
+
+
+def search_models(df, query):
+    if query:
+        return df[df['Models'].str.contains(query, case=False, na=False)]
+    return df
+
+
+# def get_size_range(df):
+#     numeric_sizes = df[df['Model Size(B)'].apply(lambda x: isinstance(x, (int, float)))]['Model Size(B)']
+#     if len(numeric_sizes) > 0:
+#         return float(numeric_sizes.min()), float(numeric_sizes.max())
+#     return 0, 1000
+
+
+def get_size_range(df):
+    sizes = df['Model Size(B)'].apply(lambda x: 1000.0 if x == 'unknown' else x)
+    return float(sizes.min()), float(sizes.max())
+
+
+def process_model_size(size):
+    if pd.isna(size) or size == 'unk':
+        return 'unknown'
+    try:
+        val = float(size)
+        return val
+    except (ValueError, TypeError):
+        return 'unknown'
+
+
+def filter_columns_by_subjects(df, selected_subjects=None):
+    if selected_subjects is None or len(selected_subjects) == 0:
+        return df[COLUMN_NAMES]
+    
+    base_columns = ['Models', 'Model Size(B)', 'Data Source', 'Overall']
+    selected_columns = base_columns + selected_subjects
+    
+    available_columns = [col for col in selected_columns if col in df.columns]
+    return df[available_columns]
+
+def get_subject_choices():
+    return SUBJECTS
+