types fix + mmluproru

data/leaderboard.json

@@ -1 +1 @@
-[{"musicmc": 0.2936170212765957, "lawmc": 0.48094747682801237, "model": "apsys/saiga_3_8b", "moviesmc": 0.3402777777777778, "booksmc": 0.3112033195020747, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.2872340425531915, "lawmc": 0.5066941297631308, "model": "vikhr-52-7b-chat-hf/apsys", "moviesmc": 0.4837962962962963, "booksmc": 0.3070539419087137, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.2936170212765957, "lawmc": 0.5345005149330587, "model": "RefalMachine/llama3 ushanka", "moviesmc": 0.35185185185185186, "booksmc": 0.3257261410788382, "model_dtype": "torch.bfloat16", "ppl": 0}, {"musicmc": 0.3021276595744681, "lawmc": 0.544799176107106, "model": "IlyaGusev/saiga_llama3_8b", "moviesmc": 0.3958333333333333, "booksmc": 0.3381742738589212, "model_dtype": "torch.bfloat16", "ppl": 0}, {"musicmc": 0.251063829787234, "lawmc": 0.48712667353244077, "model": "apsys/vikhr-52-7b", "moviesmc": 0.4212962962962963, "booksmc": 0.3112033195020747, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.24468085106382978, "lawmc": 0.4788877445932029, "model": "apsys/vikhr-53-7b-32k", "moviesmc": 0.4050925925925926, "booksmc": 0.3049792531120332, "model_dtype": "torch.float16", "ppl": 0}]
+[{"musicmc": 0.2936170212765957, "lawmc": 0.48094747682801237, "model": "apsys/saiga_3_8b", "moviesmc": 0.3402777777777778, "booksmc": 0.3112033195020747, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.2723404255319149, "lawmc": 0.4850669412976313, "model": "Nexusflow/Starling-LM-7B-beta", "moviesmc": 0.38657407407407407, "booksmc": 0.3070539419087137, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.20851063829787234, "lawmc": 0.47167868177136973, "model": "Salesforce/LLaMA-3-8B-SFR-Iterative-DPO-R", "moviesmc": 0.3055555555555556, "booksmc": 0.26141078838174275, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.3021276595744681, "lawmc": 0.544799176107106, "model": "alexwortega/saiga_submit", "moviesmc": 0.3958333333333333, "booksmc": 0.3381742738589212, "model_dtype": "torch.bfloat16", "ppl": 0}, {"musicmc": 0.2872340425531915, "lawmc": 0.5066941297631308, "model": "vikhr-52-7b-chat-hf/apsys", "moviesmc": 0.4837962962962963, "booksmc": 0.3070539419087137, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.20851063829787234, "lawmc": 0.42636457260556127, "model": "cohere/aya-8b", "moviesmc": 0.3287037037037037, "booksmc": 0.24273858921161826, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.24680851063829787, "lawmc": 0.48712667353244077, "model": "lightblue/suzume-llama-3-8B-multilingual", "moviesmc": 0.3587962962962963, "booksmc": 0.2966804979253112, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.2936170212765957, "lawmc": 0.5345005149330587, "model": "RefalMachine/llama3 ushanka", "moviesmc": 0.35185185185185186, "booksmc": 0.3257261410788382, "model_dtype": "torch.bfloat16", "ppl": 0}, {"musicmc": 0.28297872340425534, "lawmc": 0.5406797116374872, "model": "microsoft/Phi-3-medium-4k-instruct", "moviesmc": 0.42824074074074076, "booksmc": 0.3817427385892116, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.3021276595744681, "lawmc": 0.544799176107106, "model": "IlyaGusev/saiga_llama3_8b", "moviesmc": 0.3958333333333333, "booksmc": 0.3381742738589212, "model_dtype": "torch.bfloat16", "ppl": 0}, {"musicmc": 0.251063829787234, "lawmc": 0.48712667353244077, "model": "apsys/vikhr-52-7b", "moviesmc": 0.4212962962962963, "booksmc": 0.3112033195020747, "model_dtype": "torch.float16", "ppl": 0}, {"musicmc": 0.24468085106382978, "lawmc": 0.4788877445932029, "model": "apsys/vikhr-53-7b-32k", "moviesmc": 0.4050925925925926, "booksmc": 0.3049792531120332, "model_dtype": "torch.float16", "ppl": 0}]

src/display/utils.py

@@ -78,9 +78,9 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("model", "ma
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # # Model information
-auto_eval_column_dict.append(["avg", ColumnContent, ColumnContent("Type", "number", 1,0,1)])
-auto_eval_column_dict.append(["ppl", ColumnContent, ColumnContent("Type", "number", 0)])
-auto_eval_column_dict.append(["model_dtype", ColumnContent, ColumnContent("Type", "number", 0)])
+auto_eval_column_dict.append(["avg", ColumnContent, ColumnContent("avg", "number", 1,0,1)])
+auto_eval_column_dict.append(["ppl", ColumnContent, ColumnContent("ppl", "number", 0)])
+auto_eval_column_dict.append(["model_dtype", ColumnContent, ColumnContent("model_dtype", "number", 0)])
 # auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 # auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 # auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])

src/leaderboard/build_leaderboard.py

@@ -72,8 +72,10 @@ def build_leadearboard_df():
         else:
             df['mmluproru'] = 0
         leaderboard_df = df[['model','mmluproru','moviesmc','musicmc','lawmc','booksmc','model_dtype','ppl']]
-        leaderboard_df['avg'] = leaderboard_df[['moviesmc','musicmc','lawmc','booksmc','mmluproru']].mean(axis=1)
-        leaderboard_df.sort_values(by='avg',ascending=False,inplace=True)
+        leaderboard_df['avg'] = leaderboard_df[['moviesmc','musicmc','lawmc','booksmc','mmluproru']].mean(axis=1).values
+        # print(leaderboard_df.columns)
+        leaderboard_df.sort_values(by='avg',ascending=False,inplace=True,axis=0)
         numeric_cols = leaderboard_df.select_dtypes(include=['number']).columns
+        # print(numeric_cols)
         leaderboard_df[numeric_cols] = leaderboard_df[numeric_cols].round(3)
     return leaderboard_df.copy()