File size: 3,561 Bytes
9203553 0c58008 9203553 adc61b7 9203553 04f40cd 9203553 5ee35e1 f2f6f8f 8d88ab6 fe5b7b1 2d3525d 9203553 0c58008 9203553 dba91fa 94b3985 dba91fa 9025860 4ff951b a9c6e9c fe5b7b1 b241b45 0c58008 b241b45 9203553 dba91fa 9289e69 8e400bb a9c6e9c a5e37f8 b241b45 dba91fa aa8b23d 9203553 f2f6f8f b02eaf2 9025860 774056a fd10e2f fe5b7b1 0c58008 c3a2163 0c58008 c3a2163 a9273cf f2f6f8f b02eaf2 9025860 4ff951b 0c58008 c3a2163 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
from typing import List
# Common dictionary to map the columns names
COLUMNS_PRETTY = {
"bleu": "BLEU",
"chrf": "ChrF",
"rouge1": "ROUGE-1",
"rouge2": "ROUGE-2",
"rougeL": "ROUGE-L",
"bertscore": "BERTScore",
"bertscore_normalized": "BERTScore (Normalized)",
"model_name": "Model Name",
"model_availability": "Availability",
"urls": "Resources",
"context_size": "Context Size",
"submitted_by": "Submitted By",
"EM infile": "EM infile",
"EM inproject": "EM inproject",
"EM common": "EM common",
"EM commited": "EM committed",
"EM non_informative": "EM non-informative",
"EM random": "EM random",
"EM all": "EM all",
"dataset": "Dataset",
"CompScore": "CompScore",
"context": "Context",
"task_type": "Task type",
}
# Add your metrics
METRICS_PER_TASK = {
"commit_message_generation": [
"BLEU",
"ChrF",
"ROUGE-1",
"ROUGE-2",
"ROUGE-L",
"BERTScore",
"BERTScore (Normalized)",
],
"project_code_completion": [
"EM infile",
"EM inproject",
"EM common",
"EM committed",
"EM non-informative",
"EM random",
"EM all",
],
"bug_localization": [
"k",
"P@k",
"R@k",
"f1-score",
],
"module_summarization": [
"CompScore",
],
"library_based_code_generation": [
"ChrF",
"API Recall",
],
"ci_builds_repair": [
"Pass@1",
],
}
SORT_COLUMN_PER_TASK = {
"commit_message_generation": "ROUGE-1",
"project_code_completion": "EM inproject",
"bug_localization": "Model Name",
"module_summarization": "CompScore",
"library_based_code_generation": "API Recall",
"ci_builds_repair": "Pass@1",
}
def get_columns_per_task(task_id: str) -> List[str]:
metrics_per_task = METRICS_PER_TASK[task_id]
if task_id == 'project_code_completion':
return ["Model Name", "Context Size", "Dataset Name", "Dataset"] + metrics_per_task + ["Availability", "Submitted By", "Resources"]
if task_id == 'bug_localization':
return ["Model Name", "Availability", "Context Size", "Dataset"] + metrics_per_task + ["Submitted By", "Resources"]
if task_id == 'module_summarization':
return ["Model Name", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"]
if task_id == 'library_based_code_generation':
return ["Model Name", "Context"] + metrics_per_task + ["Availability", "Submitted By", "Resources"]
if task_id == 'ci_builds_repair':
return ["Model Name", "Context Size", "Task type"] + metrics_per_task + ["Availability", "Submitted By", "Resources"]
return ["Model Name", "Context Size", "Availability"] + metrics_per_task + ["Submitted By", "Resources"]
def get_types_per_task(task_id: str) -> List[str]:
metrics_per_task = METRICS_PER_TASK.get(task_id, (0, 0, 0, 0, 0))
if task_id == 'project_code_completion':
return ["html", "markdown", "markdown", "html"] + ["number" for _ in metrics_per_task] + ["markdown", "markdown", "html"]
if task_id == 'bug_localization':
return ["html", "markdown", "markdown", "html"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]
if task_id == 'ci_builds_repair':
return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "markdown", "html"]
return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]
|