|
|
from dataclasses import dataclass |
|
|
from enum import Enum |
|
|
|
|
|
@dataclass |
|
|
class EvalDimension: |
|
|
metric: str |
|
|
col_name: str |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class EvalDimensions(Enum): |
|
|
d0 = EvalDimension("speed", "Speed (words/sec)") |
|
|
d1 = EvalDimension("contamination_score", "Contamination Score") |
|
|
d2 = EvalDimension("paraphrasing", "Paraphrasing") |
|
|
d3 = EvalDimension("sentiment analysis", "Sentiment Analysis") |
|
|
d4 = EvalDimension("coding", "Coding") |
|
|
d5 = EvalDimension("function calling", "Function Calling") |
|
|
d6 = EvalDimension("rag qa", "RAG QA") |
|
|
d7 = EvalDimension("reading comprehension", "Reading Comprehension") |
|
|
d8 = EvalDimension("entity extraction", "Entity Extraction") |
|
|
d9 = EvalDimension("summarization", "Summarization") |
|
|
d10 = EvalDimension("long context", "Long Context") |
|
|
d11 = EvalDimension("mmlu", "MMLU") |
|
|
d12 = EvalDimension("arabic language & grammar", "Arabic Language & Grammar") |
|
|
d13 = EvalDimension("general knowledge", "General Knowledge") |
|
|
d14 = EvalDimension("translation (incl dialects)", "Translation (incl Dialects)") |
|
|
d15 = EvalDimension("trust & safety","Trust & Safety") |
|
|
d16 = EvalDimension("writing (incl dialects)", "Writing (incl Dialects)") |
|
|
d17 = EvalDimension("dialect detection", "Dialect Detection") |
|
|
d18 = EvalDimension("reasoning & math", "Reasoning & Math") |
|
|
d19 = EvalDimension("diacritization", "Diacritization") |
|
|
d20 = EvalDimension("instruction following", "Instruction Following") |
|
|
d21 = EvalDimension("transliteration", "Transliteration") |
|
|
d22 = EvalDimension("structuring", "Structuring") |
|
|
d23 = EvalDimension("hallucination", "Hallucination") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
NUM_FEWSHOT = 0 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
TITLE = """<div ><img class='abl_header_image' src='https://huggingface.co/spaces/silma-ai/Arabic-LLM-Broad-Leaderboard/resolve/main/src/images/abl_logo.png' ></div>""" |
|
|
|
|
|
|
|
|
INTRODUCTION_TEXT = """ |
|
|
<h1 style='width: 100%;text-align: center;' id="space-title">Arabic Broad Leaderboard (ABL) - The first comprehensive Leaderboard for Arabic LLMs</h1> |
|
|
ABL is the official Leaderboard of <a href='https://huggingface.co/datasets/silma-ai/arabic-broad-benchmark' target='_blank'>Arabic Broad Benchmark (ABB)</a>. |
|
|
With advanced features and innovative visualizations, we provide the community with a comprehensive view of the capabilities of Arabic models, showcasing their speed, diverse skills while also defending against benchmarking contamination. |
|
|
The benchmark consists of <b>450 high quality human-validated questions</b> sampled from <b>63 Arabic benchmarking datasets</b>, evaluating <b>22 categories and skills</b>. |
|
|
Find more details in the about Tab. |
|
|
|
|
|
|
|
|
""" |
|
|
|
|
|
|
|
|
LLM_BENCHMARKS_TEXT = f""" |
|
|
|
|
|
## FAQ |
|
|
|
|
|
### What is the difference betweem ABL and ABB? |
|
|
|
|
|
ABL is the Leaderboard which uses ABB benchmarking dataset and code in the backend to produce the results you see here |
|
|
|
|
|
|
|
|
### What can I learn more about ABL and ABB? |
|
|
|
|
|
Feel free to read the following resources |
|
|
ABB Page: |
|
|
ABL blog post: |
|
|
|
|
|
### How can I reproduce the results? |
|
|
|
|
|
You can easily run the ABB benchmarking code using the following command on Google Collab or your own infratructure. |
|
|
|
|
|
### What is the Benchmark Score? |
|
|
|
|
|
### What is the Contamination Score? |
|
|
|
|
|
### What is the Speed? |
|
|
|
|
|
### Why I am not allowed to submit models more than 15B parameters? |
|
|
|
|
|
|
|
|
""" |
|
|
|
|
|
EVALUATION_QUEUE_TEXT = """ |
|
|
|
|
|
""" |
|
|
|
|
|
CITATION_BUTTON_LABEL = "Copy the following snippet to cite the Leaderboard" |
|
|
CITATION_BUTTON_TEXT = r""" |
|
|
|
|
|
@misc{ABL, |
|
|
author = {SILMA AI Team}, |
|
|
title = {Arabic Broad Leaderboard}, |
|
|
year = {2025}, |
|
|
publisher = {SILMA.AI}, |
|
|
howpublished = "{\url{https://huggingface.co/spaces/silma-ai/Arabic-LLM-Broad-Leaderboard}}" |
|
|
} |
|
|
|
|
|
""" |
|
|
|
|
|
FOOTER_TEXT = """<div style='display:flex;justify-content:center;align-items:center;'> |
|
|
<span style='font-size:36px;font-weight:bold;margin-right:20px;'>Sponsored By</span> |
|
|
<a href='https://silma.ai/?ref=abl' target='_blank'> |
|
|
<img style='height:60px' src='https://huggingface.co/spaces/silma-ai/Arabic-LLM-Broad-Leaderboard/resolve/main/src/images/silma-logo-wide.png' > |
|
|
</a> |
|
|
</div>""" |
|
|
|
