Update space

app.py

@@ -96,8 +96,8 @@ def init_leaderboard(dataframe):
         interactive=False,
     )
 
-
-model_leaderboard_df = get_model_leaderboard_df()
+model_result_path = "./src/results/models_2024-10-07-14:50:12.666068.jsonl"
+model_leaderboard_df = get_model_leaderboard_df(model_result_path)
 
 def overall_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
@@ -129,7 +129,7 @@ with demo:
             
 
         with gr.TabItem("🎯 Overall", elem_id="llm-benchmark-tab-table", id=1):
-            leaderboard = overall_leaderboard(LEADERBOARD_DF)
+            leaderboard = init_leaderboard(LEADERBOARD_DF)
             
         with gr.TabItem("🔢 Math", elem_id="math-tab-table", id=2):
             

src/display/utils.py

@@ -80,9 +80,9 @@ auto_eval_column_dict.append(["revision", ColumnContent, field(default_factory=l
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 AutoEvalColumn = AutoEvalColumn()
 # print all attributes of AutoEvalColumn
-print(AutoEvalColumn.__annotations__.keys())
+# print(AutoEvalColumn.__annotations__.keys())
 # preint precision attribute
-print(AutoEvalColumn.precision)
+# print(AutoEvalColumn.precision)
 
 
 ## For the queue columns in the submission tab
@@ -144,6 +144,7 @@ class Precision(Enum):
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+# print(COLS)
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]

src/leaderboard/read_evals.py

@@ -53,7 +53,8 @@ class ModelResult:
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         
-        average = 1 / self.results[Domains.dim0.dimension] if self.results[Domains.dim0.dimension] != 0 else 0
+        # average = 1 / self.results[Domains.dim0.dimension] if self.results[Domains.dim0.dimension] != 0 else 0
+        average = 1
         # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
@@ -62,16 +63,16 @@ class ModelResult:
             AutoEvalColumn.organization.name: self.org,
             AutoEvalColumn.knowledge_cutoff.name: self.knowledge_cutoff,
 
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            # AutoEvalColumn.precision.name: self.precision.value.name,
+            # AutoEvalColumn.model_type.name: self.model_type.value.name,
+            # AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            # AutoEvalColumn.architecture.name: self.architecture,
+            # AutoEvalColumn.revision.name: self.revision,
+            # AutoEvalColumn.average.name: average,
+            # AutoEvalColumn.likes.name: self.likes,
+            # AutoEvalColumn.params.name: self.num_params,
+            # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
         for task in Tasks:
@@ -180,7 +181,7 @@ class EvalResult:
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        print(AutoEvalColumn.precision.name, self.precision.value.name)
+        # print(AutoEvalColumn.precision.name, self.precision.value.name)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,

src/populate.py

@@ -8,14 +8,18 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results, get_raw_model_results
 
 
-def get_model_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_model_leaderboard_df(results_path: str, requests_path: str="", cols: list=[], benchmark_cols: list=[]) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_model_results(results_path)
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
+    # df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    for col in cols:
+        if col not in df.columns:
+            df[col] = None
+        else:
+            df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
     # df = df[has_no_nan_values(df, benchmark_cols)]
@@ -31,7 +35,11 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
+    for col in cols:
+        if col not in df.columns:
+            df[col] = None
+        else:
+            df[col] = df[col].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]