pes

src/about.py

@@ -43,6 +43,7 @@ class Tasks(Enum):
     task27 = Task("polish_poquad_reranking", "acc,none", "poquad_reranking", "other", 0.0)
     task28 = Task("polish_abstractive_poquad_rag", "levenshtein,none", "abstractive_poquad_rag", "other", 0.0)
     task29 = Task("polish_abstractive_poquad_open_book", "levenshtein,none", "abstractive_poquad_open_book", "other", 0.0)
+    task30 = Task("polish_pes_regex", "exact_match,score-first", "pes_g", "other", 0.2)
 
 
 g_tasks = [task.value.benchmark for task in Tasks if task.value.type == "generate_until"]

src/display/utils.py

@@ -37,7 +37,7 @@ auto_eval_column_dict.append(["average_mc", ColumnContent, ColumnContent("Avg mc
 auto_eval_column_dict.append(["average_rag", ColumnContent, ColumnContent("Avg RAG", "number", True)])
 
 for task in Tasks:
-    show = task.value.col_name not in ['poquad_reranking','abstractive_poquad_rag','abstractive_poquad_open_book']
+    show = task.value.col_name not in ['poquad_reranking','abstractive_poquad_rag','abstractive_poquad_open_book', 'pes_g']
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", show)])
 # Model information