|
from typing import List |
|
|
|
|
|
COLUMNS_PRETTY = { |
|
"bleu": "BLEU", |
|
"chrf": "ChrF", |
|
"rouge1": "ROUGE-1", |
|
"rouge2": "ROUGE-2", |
|
"rougeL": "ROUGE-L", |
|
"bertscore": "BERTScore", |
|
"bertscore_normalized": "BERTScore (Normalized)", |
|
"model_name": "Model Name", |
|
"model_availability": "Availability", |
|
"urls": "Resources", |
|
"context_size": "Context Size", |
|
"submitted_by": "Submitted By", |
|
"EM infile": "EM infile", |
|
"EM inproject": "EM inproject", |
|
"EM common": "EM common", |
|
"EM commited": "EM committed", |
|
"EM non_informative": "EM non-informative", |
|
"EM random": "EM random", |
|
"EM all": "EM all", |
|
"dataset": "Dataset", |
|
"CompScore": "CompScore", |
|
"context": "Context", |
|
"task_type": "Task type", |
|
} |
|
|
|
|
|
METRICS_PER_TASK = { |
|
"commit_message_generation": [ |
|
"BLEU", |
|
"ChrF", |
|
"ROUGE-1", |
|
"ROUGE-2", |
|
"ROUGE-L", |
|
"BERTScore", |
|
"BERTScore (Normalized)", |
|
], |
|
"project_code_completion": [ |
|
"EM infile", |
|
"EM inproject", |
|
"EM common", |
|
"EM committed", |
|
"EM non-informative", |
|
"EM random", |
|
"EM all", |
|
], |
|
"bug_localization": [ |
|
"R@1", |
|
"R@2", |
|
"P@2", |
|
"f1-score", |
|
"MAP", |
|
], |
|
"module_summarization": [ |
|
"CompScore", |
|
], |
|
"library_based_code_generation": [ |
|
"ChrF", |
|
"API Recall", |
|
], |
|
"ci_builds_repair": [ |
|
"Pass@1", |
|
], |
|
} |
|
|
|
|
|
SORT_COLUMN_PER_TASK = { |
|
"commit_message_generation": "ROUGE-1", |
|
"project_code_completion": "EM inproject", |
|
"bug_localization": "Model Name", |
|
"module_summarization": "CompScore", |
|
"library_based_code_generation": "API Recall", |
|
"ci_builds_repair": "Pass@1", |
|
} |
|
|
|
|
|
def get_columns_per_task(task_id: str) -> List[str]: |
|
metrics_per_task = METRICS_PER_TASK[task_id] |
|
if task_id == 'project_code_completion': |
|
return ["Model Name", "Context Size", "Dataset Name", "Dataset"] + metrics_per_task + ["Availability", "Submitted By", "Resources"] |
|
if task_id == 'bug_localization': |
|
return ["Model Name", "Availability", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"] |
|
|
|
if task_id == 'module_summarization': |
|
return ["Model Name", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"] |
|
|
|
if task_id == 'library_based_code_generation': |
|
return ["Model Name", "Context"] + metrics_per_task + ["Availability", "Submitted By", "Resources"] |
|
|
|
if task_id == 'ci_builds_repair': |
|
return ["Model Name", "Context Size", "Task type"] + metrics_per_task + ["Availability", "Submitted By", "Resources"] |
|
|
|
return ["Model Name", "Context Size", "Availability"] + metrics_per_task + ["Submitted By", "Resources"] |
|
|
|
|
|
def get_types_per_task(task_id: str) -> List[str]: |
|
metrics_per_task = METRICS_PER_TASK.get(task_id, (0, 0, 0, 0, 0)) |
|
if task_id == 'project_code_completion': |
|
return ["html", "markdown", "markdown", "html"] + ["number" for _ in metrics_per_task] + ["markdown", "markdown", "html"] |
|
if task_id == 'bug_localization': |
|
return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"] |
|
if task_id == 'ci_builds_repair': |
|
return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "markdown", "html"] |
|
return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"] |
|
|