new tasks

src/about.py

@@ -37,9 +37,9 @@ class Tasks(Enum):
     task22 = Task("polish_polqa_open_book", "levenshtein,none", "polqa_open_book_g", "generate_until", 0.0) # generate_until
     task23 = Task("polish_polqa_closed_book", "levenshtein,none", "polqa_closed_book_g", "generate_until", 0.0) # generate_until
     task20 = Task("polish_poleval2018_task3_test_10k", "word_perplexity,none", "poleval2018_task3_test_10k", "other")
-    task24 = Task("polish_poquad_open_book", "levenshtein,none", "poquad_open_book", "other", 0.0)
-    task25 = Task("polish_eq_bench_first_turn", "first_eqbench,none", "eq_bench_first_turn", "other", 0.0)
-    task26 = Task("polish_eq_bench", "average_eqbench,none", "eq_bench", "other", 0.0)
+    # task24 = Task("polish_poquad_open_book", "levenshtein,none", "poquad_open_book", "other", 0.0)
+    # task25 = Task("polish_eq_bench_first_turn", "first_eqbench,none", "eq_bench_first_turn", "other", 0.0)
+    # task26 = Task("polish_eq_bench", "average_eqbench,none", "eq_bench", "other", 0.0)
     # task27 = Task("polish_eq_bench", "revised_eqbench,none", "eq_bench_revised", "other", 0.0)
 
 NUM_FEWSHOT = 0 # Change with your few shot

src/leaderboard/read_evals.py

@@ -428,6 +428,7 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
             print(f"not all eval values present {v.eval_name} {v.full_model}")
             continue
 
+    all_models = []
     missing_results_for_task = {}
     missing_metadata = []
     for v in eval_results.values():
@@ -441,6 +442,7 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
                     missing_results_for_task[task_name] = [f"{v.full_model}|{v.org_and_model}"]
         if r[AutoEvalColumn.lang.name] is None or r[AutoEvalColumn.lang.name] == "?":
             missing_metadata.append(f"{v.full_model}")
+        all_models.append((v.full_model, v.num_params))
 
     # print('missing_results_for_task', missing_results_for_task)
     for task, models in missing_results_for_task.items():
@@ -455,4 +457,9 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
         print(model)
     print()
 
+    print(f"All models:")
+    for model in all_models:
+        print(model)
+    print()
+
     return results