tasks

src/about.py

@@ -43,7 +43,64 @@ class Tasks(Enum):
     # task27 = Task("polish_poquad_reranking", "acc,none", "poquad_reranking", "other", 0.0)
     # task28 = Task("polish_abstractive_poquad_rag", "levenshtein,none", "abstractive_poquad_rag", "other", 0.0)
     # task29 = Task("polish_abstractive_poquad_open_book", "levenshtein,none", "abstractive_poquad_open_book", "other", 0.0)
-    task30 = Task("polish_pes_regex", "exact_match,score-first", "pes_g", "generate_until", 0.2)
+    task30a = Task("polish_pes_regex", "exact_match,score-first", "pes", "generate_until", 0.2)
+    task30 = Task("polish_pes_alergologia", "exact_match,score-first", "alergologia", "generate_until", 0.2)
+    task31 = Task("polish_pes_anestezjologia", "exact_match,score-first", "anestezjologia", "generate_until", 0.2)
+    task32 = Task("polish_pes_angiologia", "exact_match,score-first", "angiologia", "generate_until", 0.2)
+    task33 = Task("polish_pes_balneologia_i_medycyna_fizykalna", "exact_match,score-first", "balneologia_i_medycyna_fizykalna", "generate_until", 0.2)
+    task34 = Task("polish_pes_chirurgia_dziecieca", "exact_match,score-first", "chirurgia_dziecieca", "generate_until", 0.2)
+    task35 = Task("polish_pes_chirurgia_naczyniowa", "exact_match,score-first", "chirurgia_naczyniowa", "generate_until", 0.2)
+    task36 = Task("polish_pes_chirurgia_ogolna", "exact_match,score-first", "chirurgia_ogolna", "generate_until", 0.2)
+    task37 = Task("polish_pes_chirurgia_onkologiczna", "exact_match,score-first", "chirurgia_onkologiczna", "generate_until", 0.2)
+    task38 = Task("polish_pes_chirurgia_stomatologiczna", "exact_match,score-first", "chirurgia_stomatologiczna", "generate_until", 0.2)
+    task39 = Task("polish_pes_chirurgia_szczekowo-twarzowa", "exact_match,score-first", "chirurgia_szczekowo-twarzowa", "generate_until", 0.2)
+    task40 = Task("polish_pes_choroby_pluc", "exact_match,score-first", "choroby_pluc", "generate_until", 0.2)
+    task41 = Task("polish_pes_choroby_pluc_dzieci", "exact_match,score-first", "choroby_pluc_dzieci", "generate_until", 0.2)
+    task42 = Task("polish_pes_choroby_wewnetrzne", "exact_match,score-first", "choroby_wewnetrzne", "generate_until", 0.2)
+    task43 = Task("polish_pes_choroby_zakazne", "exact_match,score-first", "choroby_zakazne", "generate_until", 0.2)
+    task44 = Task("polish_pes_dermatologia_i_wenerologia", "exact_match,score-first", "dermatologia_i_wenerologia", "generate_until", 0.2)
+    task45 = Task("polish_pes_diabetologia", "exact_match,score-first", "diabetologia", "generate_until", 0.2)
+    task46 = Task("polish_pes_endokrynologia", "exact_match,score-first", "endokrynologia", "generate_until", 0.2)
+    task47 = Task("polish_pes_endokrynologia_ginekologiczna_i_rozrodczosc", "exact_match,score-first", "endokrynologia_ginekologiczna_i_rozrodczosc", "generate_until", 0.2)
+    task48 = Task("polish_pes_endokrynologia_i_diabetologia_dziecieca", "exact_match,score-first", "endokrynologia_i_diabetologia_dziecieca", "generate_until", 0.2)
+    task49 = Task("polish_pes_gastroenterologia", "exact_match,score-first", "gastroenterologia", "generate_until", 0.2)
+    task50 = Task("polish_pes_gastroenterologia_dziecieca", "exact_match,score-first", "gastroenterologia_dziecieca", "generate_until", 0.2)
+    task51 = Task("polish_pes_geriatria", "exact_match,score-first", "geriatria", "generate_until", 0.2)
+    task52 = Task("polish_pes_ginekologia_onkologiczna", "exact_match,score-first", "ginekologia_onkologiczna", "generate_until", 0.2)
+    task53 = Task("polish_pes_hematologia", "exact_match,score-first", "hematologia", "generate_until", 0.2)
+    task54 = Task("polish_pes_hipertensjologia", "exact_match,score-first", "hipertensjologia", "generate_until", 0.2)
+    task55 = Task("polish_pes_kardiochirurgia", "exact_match,score-first", "kardiochirurgia", "generate_until", 0.2)
+    task56 = Task("polish_pes_kardiologia", "exact_match,score-first", "kardiologia", "generate_until", 0.2)
+    task57 = Task("polish_pes_medycyna_pracy", "exact_match,score-first", "medycyna_pracy", "generate_until", 0.2)
+    task58 = Task("polish_pes_medycyna_paliatywna", "exact_match,score-first", "medycyna_paliatywna", "generate_until", 0.2)
+    task59 = Task("polish_pes_medycyna_ratunkowa", "exact_match,score-first", "medycyna_ratunkowa", "generate_until", 0.2)
+    task60 = Task("polish_pes_medycyna_rodzinna", "exact_match,score-first", "medycyna_rodzinna", "generate_until", 0.2)
+    task61 = Task("polish_pes_medycyna_sportowa", "exact_match,score-first", "medycyna_sportowa", "generate_until", 0.2)
+    task62 = Task("polish_pes_nefrologia", "exact_match,score-first", "nefrologia", "generate_until", 0.2)
+    task63 = Task("polish_pes_neonatologia", "exact_match,score-first", "neonatologia", "generate_until", 0.2)
+    task64 = Task("polish_pes_neurochirurgia", "exact_match,score-first", "neurochirurgia", "generate_until", 0.2)
+    task65 = Task("polish_pes_neurologia", "exact_match,score-first", "neurologia", "generate_until", 0.2)
+    task66 = Task("polish_pes_neurologia_dziecieca", "exact_match,score-first", "neurologia_dziecieca", "generate_until", 0.2)
+    task67 = Task("polish_pes_okulistyka", "exact_match,score-first", "okulistyka", "generate_until", 0.2)
+    task68 = Task("polish_pes_onkologia_kliniczna", "exact_match,score-first", "onkologia_kliniczna", "generate_until", 0.2)
+    task69 = Task("polish_pes_ortodoncja", "exact_match,score-first", "ortodoncja", "generate_until", 0.2)
+    task70 = Task("polish_pes_ortopedia", "exact_match,score-first", "ortopedia", "generate_until", 0.2)
+    task71 = Task("polish_pes_otolaryngologia", "exact_match,score-first", "otolaryngologia", "generate_until", 0.2)
+    task72 = Task("polish_pes_patomorfologia", "exact_match,score-first", "patomorfologia", "generate_until", 0.2)
+    task73 = Task("polish_pes_pediatria", "exact_match,score-first", "pediatria", "generate_until", 0.2)
+    task74 = Task("polish_pes_perinatologia", "exact_match,score-first", "perinatologia", "generate_until", 0.2)
+    task75 = Task("polish_pes_periodontologia", "exact_match,score-first", "periodontologia", "generate_until", 0.2)
+    task76 = Task("polish_pes_poloznictwo_i_ginekologia", "exact_match,score-first", "poloznictwo_i_ginekologia", "generate_until", 0.2)
+    task77 = Task("polish_pes_protetyka_stomatologiczna", "exact_match,score-first", "protetyka_stomatologiczna", "generate_until", 0.2)
+    task78 = Task("polish_pes_psychiatria", "exact_match,score-first", "psychiatria", "generate_until", 0.2)
+    task79 = Task("polish_pes_psychiatria_dzieci_i_mlodziezy", "exact_match,score-first", "psychiatria_dzieci_i_mlodziezy", "generate_until", 0.2)
+    task80 = Task("polish_pes_radiologia_i_diagnostyka_obrazowa", "exact_match,score-first", "radiologia_i_diagnostyka_obrazowa", "generate_until", 0.2)
+    task81 = Task("polish_pes_radioterapia_onkologiczna", "exact_match,score-first", "radioterapia_onkologiczna", "generate_until", 0.2)
+    task82 = Task("polish_pes_rehabilitacja_medyczna", "exact_match,score-first", "rehabilitacja_medyczna", "generate_until", 0.2)
+    task83 = Task("polish_pes_reumatologia", "exact_match,score-first", "reumatologia", "generate_until", 0.2)
+    task84 = Task("polish_pes_stomatologia_dziecieca", "exact_match,score-first", "stomatologia_dziecieca", "generate_until", 0.2)
+    task85 = Task("polish_pes_stomatologia_zachowawcza", "exact_match,score-first", "stomatologia_zachowawcza", "generate_until", 0.2)
+    task86 = Task("polish_pes_transplantologia_kliniczna", "exact_match,score-first", "transplantologia_kliniczna", "generate_until", 0.2)
 
 
 g_tasks = [task.value.benchmark for task in Tasks if task.value.type == "generate_until"]

src/display/utils.py

@@ -32,14 +32,14 @@ auto_eval_column_dict.append(["n_shot", ColumnContent, ColumnContent("n_shot", "
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", True)])
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-auto_eval_column_dict.append(["average_old", ColumnContent, ColumnContent("Average old", "number", False)])
-auto_eval_column_dict.append(["average_g", ColumnContent, ColumnContent("Avg g", "number", True)])
-auto_eval_column_dict.append(["average_mc", ColumnContent, ColumnContent("Avg mc", "number", True)])
-auto_eval_column_dict.append(["average_rag", ColumnContent, ColumnContent("Avg RAG", "number", True)])
+# auto_eval_column_dict.append(["average_old", ColumnContent, ColumnContent("Average old", "number", False)])
+# auto_eval_column_dict.append(["average_g", ColumnContent, ColumnContent("Avg g", "number", True)])
+# auto_eval_column_dict.append(["average_mc", ColumnContent, ColumnContent("Avg mc", "number", True)])
+# auto_eval_column_dict.append(["average_rag", ColumnContent, ColumnContent("Avg RAG", "number", True)])
 
 for task in Tasks:
-    show = task.value.col_name not in ['poquad_reranking','abstractive_poquad_rag','abstractive_poquad_open_book', 'pes_g']
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", show)])
+    # show = task.value.col_name not in ['poquad_reranking','abstractive_poquad_rag','abstractive_poquad_open_book', 'pes_g']
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
 
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])

src/leaderboard/read_evals.py

@@ -283,30 +283,30 @@ class EvalResult:
         except AttributeError:
             print(f"AttributeError revision")
 
-        try:
-            data_dict[AutoEvalColumn.average_old.name] = average_old
-        except KeyError:
-            print(f"Could not find average_old")
+        # try:
+        #     data_dict[AutoEvalColumn.average_old.name] = average_old
+        # except KeyError:
+        #     print(f"Could not find average_old")
 
         try:
             data_dict[AutoEvalColumn.average.name] = average
         except KeyError:
             print(f"Could not find average")
 
-        try:
-            data_dict[AutoEvalColumn.average_g.name] = average_g
-        except KeyError:
-            print(f"Could not find average_g")
+        # try:
+        #     data_dict[AutoEvalColumn.average_g.name] = average_g
+        # except KeyError:
+        #     print(f"Could not find average_g")
 
-        try:
-            data_dict[AutoEvalColumn.average_mc.name] = average_mc
-        except KeyError:
-            print(f"Could not find average_mc")
+        # try:
+        #     data_dict[AutoEvalColumn.average_mc.name] = average_mc
+        # except KeyError:
+        #     print(f"Could not find average_mc")
 
-        try:
-            data_dict[AutoEvalColumn.average_rag.name] = average_rag
-        except KeyError:
-            print(f"Could not find average_rag")
+        # try:
+        #     data_dict[AutoEvalColumn.average_rag.name] = average_rag
+        # except KeyError:
+        #     print(f"Could not find average_rag")
 
         try:
             data_dict[AutoEvalColumn.license.name] = self.license