update

src/display/utils.py

@@ -26,6 +26,7 @@ class AutoEvalColumn:  # Auto evals column
     model_type_symbol = ColumnContent("T", "str", True, never_hidden=True)
     model = ColumnContent("Model", "markdown", True, never_hidden=True)
     average = ColumnContent("Average ⬆️", "number", True)
+
     arc = ColumnContent("ARC", "number", True)
     hellaswag = ColumnContent("HellaSwag", "number", True)
     mmlu = ColumnContent("MMLU", "number", True)
@@ -33,6 +34,8 @@ class AutoEvalColumn:  # Auto evals column
     winogrande = ColumnContent("Winogrande", "number", True)
     gsm8k = ColumnContent("GSM8K", "number", True)
     drop = ColumnContent("DROP", "number", True)
+    nqopen = ColumnContent("NQ Open", "number", True)
+
     model_type = ColumnContent("Type", "str", False)
     architecture = ColumnContent("Architecture", "str", False)
     weight_type = ColumnContent("Weight type", "str", False, True)

src/populate.py

@@ -10,6 +10,10 @@ from src.leaderboard.read_evals import get_raw_eval_results
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    # Returns a list of EvalResult
+    # raw_data[0]:
+    # EvalResult(eval_name='EleutherAI_pythia-1.3b_torch.float16', full_model='EleutherAI/pythia-1.3b', org='EleutherAI', model='pythia-1.3b', revision='34b668ff0acfe56f2d541aa46b385557ee39eb3f', results={'arc:challenge': 31.14334470989761, 'hellaswag': 51.43397729535949, 'hendrycksTest': 26.55151159544371, 'truthfulqa:mc': 39.24322830092449, 'winogrande': 57.37963693764798, 'gsm8k': 0.9855951478392722, 'drop': 4.056312919463095}, precision='torch.float16', model_type=<ModelType.PT: ModelTypeDetails(name='pretrained', symbol='🟢')>, weight_type='Original', architecture='GPTNeoXForCausalLM', license='apache-2.0', likes=7, num_params=1.312, date='2023-09-09T10:52:17Z', still_on_hub=True)
+    # EvalResult and get_raw_eval_results are defined in ./src/leaderboard/read_evals.py, the results slots are not hardcoded
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
     all_data_json.append(baseline_row)