types fix + mmluproru

app.py

@@ -185,7 +185,7 @@ def update_board():
     # shutil.rmtree("./data")
     download_dataset("Vikhrmodels/s-openbench-eval", "m_data")
     import glob
-    data_list = [{"musicmc": 0.3021276595744681, "lawmc": 0.2800829875518672, "model": "apsys/saiga_3_8b", "moviesmc": 0.3472222222222222, "booksmc": 0.2800829875518672, "model_dtype": "torch.float16", "ppl": 0}]
+    data_list = [{"musicmc": 0.3021276595744681, "lawmc": 0.2800829875518672, "model": "apsys/saiga_3_8b", "moviesmc": 0.3472222222222222, "booksmc": 0.2800829875518672, "model_dtype": "torch.float16", "ppl": 0, 'mmluproru':0}]
     for file in glob.glob("./m_data/model_data/external/*.json"):
         with open(file) as f:
 

src/display/utils.py

@@ -53,6 +53,7 @@ class Tasks(Enum):
     movies = Task("moviesmc", "acc", "moviesmc")
     music = Task("musicmc", "acc", "musicmc")
     law = Task("lawmc", "acc", "lawmc")
+    mmluproru = Task("mmluproru", "acc", "mmluproru")
 
 
 # These classes are for user facing column names,
@@ -77,7 +78,7 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("model", "ma
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # # Model information
-auto_eval_column_dict.append(["avg", ColumnContent, ColumnContent("Type", "number", 0)])
+auto_eval_column_dict.append(["avg ⬆️", ColumnContent, ColumnContent("Type", "number", 1,0,1)])
 auto_eval_column_dict.append(["ppl", ColumnContent, ColumnContent("Type", "number", 0)])
 auto_eval_column_dict.append(["model_dtype", ColumnContent, ColumnContent("Type", "number", 0)])
 # auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])

src/leaderboard/build_leaderboard.py

@@ -66,9 +66,13 @@ def build_leadearboard_df():
     with open(f"{os.path.abspath(DATA_PATH)}/leaderboard.json", "r", encoding="utf-8") as eval_file:
         f=json.load(eval_file)
         print(f)
-
-        leaderboard_df = pd.DataFrame.from_records(f)[['model','moviesmc','musicmc','lawmc','booksmc','model_dtype','ppl']]
-        leaderboard_df['avg'] = leaderboard_df[['moviesmc','musicmc','lawmc','booksmc']].mean(axis=1)
+        df = pd.DataFrame.from_records(f)
+        if 'mmluproru' in list(df.columns):
+            df['mmluproru'] = df['mmluproru'].fillna(0)
+        else:
+            df['mmluproru'] = 0
+        leaderboard_df = [['model','mmluproru','moviesmc','musicmc','lawmc','booksmc','model_dtype','ppl']]
+        leaderboard_df['avg'] = leaderboard_df[['moviesmc','musicmc','lawmc','booksmc','mmluproru']].mean(axis=1)
         numeric_cols = leaderboard_df.select_dtypes(include=['number']).columns
         leaderboard_df[numeric_cols] = leaderboard_df[numeric_cols].round(3)
     return leaderboard_df.copy()