labels_v2.py

TASKS_V2 = ['V-CLS', 'V-QA', 'V-RET', 'V-MRET']

COLUMN_NAMES_V2 = [
    "Rank", "Models", "Model Size(B)", "Data Source",
    "V2-Overall",
    "V-CLS", "V-QA", "V-RET", "V-MRET", "VisDoc"
]

BASE_COLS_V2 = [col for col in COLUMN_NAMES_V2 if col not in TASKS_V2]

DATA_TITLE_TYPE_V2 = ['number', 'markdown', 'str', 'markdown', 'number', 'number', 'number', 'number', 'number', 'number']

TABLE_INTRODUCTION_V2 = """MMEB-V2"""

CITATION_BUTTON_TEXT_V2 = r"""TBA"""

SUBMIT_INTRODUCTION_V2 = """# Submit on MMEB-V2 Leaderboard Introduction

## ⚠ Please note that you need to submit the JSON file with the following format:
```json
[
    {
        "Model": "<Model Name>",
        <Optional>"URL": "<Model URL>",
        "Model Size(B)": 1000,
        "Data Source": Self-Reported,
        "V2-Overall": 50.0,
        "V-CLS": 50.0,
        "V-QA": 50.0,
        "V-RET": 50.0,
        "V-MRET": 50.0, 
        "VisDoc": 50.0
    }, 
]
```
You may refer to the Github page for instructions about evaluating your model. 
Github link: https://github.com/TIGER-AI-Lab/VLM2Vec. \n
Please send us an email at m7su@uwaterloo.ca, attaching the JSON file. We will review your submission and update the leaderboard accordingly.
"""