long-code-arena / src /leaderboard_formatting.py
saridormi's picture
Make files submissions instructions task-specific & other small changes
6c92442
raw
history blame
888 Bytes
from typing import List
COLUMNS_PRETTY = {
"bleu": "BLEU",
"chrf": "ChrF",
"rouge1": "ROUGE-1",
"rouge2": "ROUGE-2",
"rougeL": "ROUGE-L",
"bertscore": "BERTScore",
"bertscore_normalized": "BERTScore (Normalized)",
"model_name": "Model Name",
"model_availability": "Availability",
"urls": "URLs",
"context_size": "Context Size",
"submitted_by": "Submitted By",
}
METRICS_PER_TASK = {
"commit_message_generation": [
"BLEU",
"ChrF",
"ROUGE-1",
"ROUGE-2",
"ROUGE-L",
"BERTScore",
"BERTScore (Normalized)",
]
}
SORT_COLUMN_PER_TASK = {"commit_message_generation": "ROUGE-1"}
def get_columns_per_task(task_id: str) -> List[str]:
metrics_per_task = METRICS_PER_TASK[task_id]
return ["Model Name", "Availability", "Context Size"] + metrics_per_task + ["Submitted By"]