src/leaderboard_formatting.py

from typing import List

# Common dictionary to map the columns names
COLUMNS_PRETTY = {
    "bleu": "BLEU",
    "chrf": "ChrF",
    "rouge1": "ROUGE-1",
    "rouge2": "ROUGE-2",
    "rougeL": "ROUGE-L",
    "bertscore": "BERTScore",
    "bertscore_normalized": "BERTScore (Normalized)",
    "model_name": "Model Name",
    "model_availability": "Availability",
    "urls": "Resources",
    "context_size": "Context Size",
    "submitted_by": "Submitted By",
    "EM infile": "EM infile",
    "EM inproject": "EM inproject",
    "EM common": "EM common",
    "EM commited": "EM committed",
    "EM non_informative": "EM non-informative",
    "EM random": "EM random",
    "EM all": "EM all",    
    "dataset": "Dataset",
    "CompScore": "CompScore",
    "context": "Context",
    "task_type": "Task type",
}

# Add your metrics
METRICS_PER_TASK = {
    "commit_message_generation": [
        "BLEU",
        "ChrF",
        "ROUGE-1",
        "ROUGE-2",
        "ROUGE-L",
        "BERTScore",
        "BERTScore (Normalized)",
    ],
    "project_code_completion": [
        "EM infile",
        "EM inproject",
        "EM common",
        "EM committed",
        "EM non-informative",
        "EM random",
        "EM all",
    ],
    "bug_localization": [
        "R@1",
        "R@2",
        "P@2",
        "f1-score",
        "MAP",
    ],
    "module_summarization": [
        "CompScore",
    ],
    "library_based_code_generation": [
        "ChrF",
        "API Recall",
    ],
    "ci_builds_repair": [
        "Pass@1",
    ],
}


SORT_COLUMN_PER_TASK = {
    "commit_message_generation": "ROUGE-1",
    "project_code_completion": "EM inproject",
    "bug_localization": "Model Name",
    "module_summarization": "CompScore",
    "library_based_code_generation": "API Recall",
    "ci_builds_repair": "Pass@1",
}


def get_columns_per_task(task_id: str) -> List[str]:
    metrics_per_task = METRICS_PER_TASK[task_id]
    if task_id == 'project_code_completion':
        return ["Model Name", "Context Size", "Dataset Name", "Dataset"] + metrics_per_task + ["Availability", "Submitted By", "Resources"]
    if task_id == 'bug_localization':
        return ["Model Name", "Availability", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"]
    
    if task_id == 'module_summarization':
        return ["Model Name", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"]

    if task_id == 'library_based_code_generation':
        return ["Model Name", "Context"] + metrics_per_task + ["Availability", "Submitted By", "Resources"]

    if task_id == 'ci_builds_repair':
        return ["Model Name", "Context Size", "Task type"] + metrics_per_task + ["Availability", "Submitted By", "Resources"]

    return ["Model Name", "Context Size", "Availability"] + metrics_per_task + ["Submitted By", "Resources"]


def get_types_per_task(task_id: str) -> List[str]:
    metrics_per_task = METRICS_PER_TASK.get(task_id, (0, 0, 0, 0, 0))
    if task_id == 'project_code_completion':
        return ["html", "markdown", "markdown", "html"] + ["number" for _ in metrics_per_task] + ["markdown", "markdown", "html"]
    if task_id == 'bug_localization':
        return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]
    if task_id == 'ci_builds_repair':
        return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "markdown", "html"]
    return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]