update

src/display/css_html_js.py

@@ -34,7 +34,7 @@ custom_css = """
 }
 
 #leaderboard-table table td:first-child {
-    text-align: right;
+    text-align: right !important;
 }
 
 #leaderboard-table-lite {

src/display/utils.py

@@ -16,14 +16,6 @@ class Task:
 
 
 class Tasks(Enum):
-    # arc = Task("arc:challenge", "acc_norm", "ARC")
-    # hellaswag = Task("hellaswag", "acc_norm", "HellaSwag")
-    # mmlu = Task("hendrycksTest", "acc", "MMLU")
-    # truthfulqa = Task("truthfulqa:mc", "mc2", "TruthfulQA")
-    # winogrande = Task("winogrande", "acc", "Winogrande")
-    # gsm8k = Task("gsm8k", "acc", "GSM8K")
-    # drop = Task("drop", "f1", "DROP")
-
     nqopen = Task("nq8", "em", "NQ Open/EM")
     triviaqa = Task("tqa8", "em", "TriviaQA/EM")
 
@@ -40,7 +32,7 @@ class Tasks(Enum):
     cnndm_b = Task("cnndm_v2", "bertscore_precision", "CNN-DM/BERT-P")
 
     race = Task("race", "acc", "RACE/Acc")
-    # squadv2 = Task("squadv2", "exact_normalised", "SQUaDv2/EM")
+    squadv2 = Task("squadv2", "exact", "SQUaDv2/EM")
 
     memotrap = Task("memo-trap_v2", "acc", "MemoTrap/Acc")
     ifeval = Task("ifeval", "prompt_level_strict_acc", "IFEval/Acc")
@@ -69,10 +61,13 @@ auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avg", "number", True)])
+# auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avg", "number", True)])
+
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])

src/leaderboard/read_evals.py

@@ -4,7 +4,7 @@ import os
 from dataclasses import dataclass
 
 import dateutil
-import numpy as np
+# import numpy as np
 
 from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
@@ -22,11 +22,11 @@ def is_float(string):
 @dataclass
 class EvalResult:
     # Also see src.display.utils.AutoEvalColumn for what will be displayed.
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
     org: str 
     model: str
-    revision: str # commit hash, "" if main
+    revision: str  # commit hash, "" if main
     results: dict
     precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
@@ -72,55 +72,41 @@ class EvalResult:
                 architecture = ";".join(architectures)
 
         # Extract results available in this file (some results are split in several files)
-        results = {}
-
-        task_iterator = Tasks
-        if is_backend is True:
-            from src.backend.envs import Tasks as BackendTasks
-            task_iterator = BackendTasks
-
-        for task in task_iterator:
-            task = task.value
-
-            def post_process_results(results: dict) -> dict:
-                # {'nq_open': {'em': 0.018005540166204988, 'em_stderr': 0.0022134216580395583}}
-                res_copy = results.copy()
-
-                for task_name in res_copy.keys():
-                    entry_copy = results[task_name].copy()
-
-                    for k, v in entry_copy.items():
-                        if "exact_match" in k:
-                            results[task_name][k.replace("exact_match", "em")] = v
-                        if "squadv2" in task_name:
-                            value = results[task_name][k]
-                            if is_float(value) and 'normalised' not in k:
-                                results[task_name][f"{k}_normalised"] = value / 100.0
-                            else:
-                                del results[task_name][k]
-
-                    entry_copy = results[task_name].copy()
-
-                    for k, v in entry_copy.items():
-                        if "," in k:
-                            tokens = k.split(",")
-                            results[task_name][tokens[0]] = v
-
-                return results
 
-            accs = np.array([v.get(task.metric, None) for k, v in post_process_results(data["results"]).items() if task.benchmark in k])
+        # data['results'] is {'nq_open': {'em': 0.24293628808864265, 'em_stderr': 0.007138697341112125}}
 
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            # print(accs)
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return EvalResult(eval_name=result_key, full_model=full_model, org=org, model=model, results=results,
-                          precision=precision, revision=config.get("model_sha", ""), still_on_hub=still_on_hub,
-                          architecture=architecture)
+        results = {}
+        for benchmark, benchmark_results in data['results'].items():
+            if benchmark not in results:
+                results[benchmark] = {}
+
+            for metric, value in benchmark_results.items():
+                to_add = True
+                if '_stderr' in metric:
+                    to_add = False
+                if 'alias' in metric:
+                    to_add = False
+
+                if ',' in metric:
+                    metric = metric.split(',')[0]
+                metric = metric.replace("exact_match", "em")
+
+                if to_add is True:
+                    multiplier = 100.0
+                    if 'rouge' in metric:
+                        multiplier = 1.0
+                    if 'squad' in benchmark:
+                        multiplier = 1.0
+
+                    # print('RESULTS', data['results'])
+                    # print('XXX', benchmark, metric, value, multiplier)
+                    results[benchmark][metric] = value * multiplier
+
+        res = EvalResult(eval_name=result_key, full_model=full_model, org=org, model=model, results=results,
+                         precision=precision, revision=config.get("model_sha", ""), still_on_hub=still_on_hub,
+                         architecture=architecture)
+
+        return res
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
@@ -129,6 +115,7 @@ class EvalResult:
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
+
             self.model_type = ModelType.from_str(request.get("model_type", ""))
             self.weight_type = WeightType[request.get("weight_type", "Original")]
             self.license = request.get("license", "?")
@@ -146,7 +133,10 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+
+        # breakpoint()
+        # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -157,7 +147,7 @@ class EvalResult:
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.dummy.name: self.full_model,
             AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
+            # AutoEvalColumn.average.name: average,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
@@ -165,7 +155,7 @@ class EvalResult:
         }
 
         for task in Tasks:
-            if task.value.benchmark in self.results:  # XXX
+            if task.value.benchmark in self.results:
                 data_dict[task.value.col_name] = self.results[task.value.benchmark]
 
         return data_dict
@@ -182,14 +172,11 @@ def get_request_file_for_model(requests_path, model_name, precision):
     # Select correct request file (precision)
     request_file = ""
     request_files = sorted(request_files, reverse=True)
-    # print('XXX', request_files)
+
     for tmp_request_file in request_files:
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
-            if (
-                # req_content["status"] in ["FINISHED", "RUNNING"] and
-                req_content["precision"] == precision.split(".")[-1]
-            ):
+            if req_content["precision"] == precision.split(".")[-1]:
                 request_file = tmp_request_file
     return request_file
 

src/populate.py

@@ -1,6 +1,7 @@
 import json
 import os
 
+import copy
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
@@ -8,24 +9,58 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.filter_models import filter_models
 from src.leaderboard.read_evals import get_raw_eval_results, EvalResult
 
+from src.backend.envs import Tasks as BackendTasks
+from src.display.utils import Tasks
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> tuple[list[EvalResult], pd.DataFrame]:
+
+def get_leaderboard_df(results_path: str,
+                       requests_path: str,
+                       cols: list,
+                       benchmark_cols: list,
+                       is_backend: bool = False) -> tuple[list[EvalResult], pd.DataFrame]:
     # Returns a list of EvalResult
-    # raw_data[0]:
-    # EvalResult(eval_name='EleutherAI_pythia-1.3b_torch.float16', full_model='EleutherAI/pythia-1.3b', org='EleutherAI', model='pythia-1.3b', revision='34b668ff0acfe56f2d541aa46b385557ee39eb3f', results={'arc:challenge': 31.14334470989761, 'hellaswag': 51.43397729535949, 'hendrycksTest': 26.55151159544371, 'truthfulqa:mc': 39.24322830092449, 'winogrande': 57.37963693764798, 'gsm8k': 0.9855951478392722, 'drop': 4.056312919463095}, precision='torch.float16', model_type=<ModelType.PT: ModelTypeDetails(name='pretrained', symbol='🟢')>, weight_type='Original', architecture='GPTNeoXForCausalLM', license='apache-2.0', likes=7, num_params=1.312, date='2023-09-09T10:52:17Z', still_on_hub=True)
-    # EvalResult and get_raw_eval_results are defined in ./src/leaderboard/read_evals.py, the results slots are not hardcoded
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data if v.is_complete()]
+    raw_data: list[EvalResult] = get_raw_eval_results(results_path, requests_path)
+
+    all_data_json_ = [v.to_dict() for v in raw_data if v.is_complete()]
+
+    name_to_bm_map = {}
+
+    task_iterator = Tasks
+    if is_backend is True:
+        task_iterator = BackendTasks
+
+    for task in task_iterator:
+        task = task.value
+        name = task.col_name
+        bm = (task.benchmark, task.metric)
+        name_to_bm_map[name] = bm
+
+    # bm_to_name_map = {bm: name for name, bm in name_to_bm_map.items()}
+
+    all_data_json = []
+    for entry in all_data_json_:
+        new_entry = copy.deepcopy(entry)
+
+        for k, v in entry.items():
+            if k in name_to_bm_map:
+                benchmark, metric = name_to_bm_map[k]
+                new_entry[k] = entry[k][metric]
+
+        all_data_json += [new_entry]
+
     # all_data_json.append(baseline_row)
     filter_models(all_data_json)
 
     df = pd.DataFrame.from_records(all_data_json)
-    if AutoEvalColumn.average.name in df:
-        df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-        df = df[cols].round(decimals=2)
 
-        # filter out if any of the benchmarks have not been produced
-        df = df[has_no_nan_values(df, benchmark_cols)]
+    # if AutoEvalColumn.average.name in df:
+    #     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+
     return raw_data, df