Update space

app.py

@@ -164,44 +164,45 @@ with demo:
             """
             gr.Markdown(DESCRIPTION_TEXT, elem_classes="markdown-text")
 
-            with gr.TabItem("Sort by Rank", elem_id="overall_sort_by_rank_subtab", id=0, elem_classes="subtab"): 
+            with gr.TabItem("⭐ Sort by Rank", elem_id="overall_sort_by_rank_subtab", id=0, elem_classes="subtab"): 
                 leaderboard = overall_leaderboard(
                     get_model_leaderboard_df(
                         model_result_path,
                         benchmark_cols=[
                             # AutoEvalColumn.rank_overall.name,
                             AutoEvalColumn.model.name, 
-                            AutoEvalColumn.rank_overall.name,
+                            
                             AutoEvalColumn.rank_math_algebra.name,
                             AutoEvalColumn.rank_math_geometry.name,
                             AutoEvalColumn.rank_math_probability.name,
                             AutoEvalColumn.rank_reason_logical.name,
                             AutoEvalColumn.rank_reason_social.name,
                             AutoEvalColumn.rank_chemistry.name,
+                            AutoEvalColumn.rank_overall.name,
                             # AutoEvalColumn.rank_cpp.name,
                             ],
-                        rank_col=[],
+                        rank_col=['sort_by_rank', 1, 8],
                     )
                 )
             
-            with gr.TabItem("Sort by Score", elem_id="overall_sort_by_score_subtab", id=1, elem_classes="subtab"): 
+            with gr.TabItem("⭐ Sort by Score", elem_id="overall_sort_by_score_subtab", id=1, elem_classes="subtab"): 
                 leaderboard = overall_leaderboard(
                     get_model_leaderboard_df(
                         model_result_path,
                         benchmark_cols=[
                             # AutoEvalColumn.rank_overall.name,
                             AutoEvalColumn.model.name, 
-                            AutoEvalColumn.license.name,
-                            AutoEvalColumn.organization.name,
-                            AutoEvalColumn.knowledge_cutoff.name,
+                            # AutoEvalColumn.license.name,
+                            # AutoEvalColumn.organization.name,
+                            # AutoEvalColumn.knowledge_cutoff.name,
 
-                            AutoEvalColumn.score_overall.name,
                             AutoEvalColumn.score_math_algebra.name,
                             AutoEvalColumn.score_math_geometry.name,
                             AutoEvalColumn.score_math_probability.name,
                             AutoEvalColumn.score_reason_logical.name,
                             AutoEvalColumn.score_reason_social.name,
                             AutoEvalColumn.score_chemistry.name,
+                            AutoEvalColumn.score_overall.name,
                             # AutoEvalColumn.score_cpp.name,
                             
                             # AutoEvalColumn.rank_overall.name,
@@ -213,7 +214,7 @@ with demo:
                             # AutoEvalColumn.rank_chemistry.name,
                             # AutoEvalColumn.rank_cpp.name,
                             ],
-                        rank_col=['sort_by_score'],
+                        rank_col=['sort_by_score', 1, 8],
                     )
                 )
             
@@ -236,26 +237,50 @@ with demo:
             gr.Markdown(DESCRIPTION_TEXT, elem_classes="markdown-text")
 
             # leaderboard = init_leaderboard(LEADERBOARD_DF)
-            with gr.TabItem("Overall", elem_id="math_overall_subtab", id=0, elem_classes="subtab"): 
-                leaderboard = overall_leaderboard(
-                    get_model_leaderboard_df(
-                        model_result_path,
-                        benchmark_cols=[
-                            AutoEvalColumn.model.name, 
-                            AutoEvalColumn.license.name,
-                            AutoEvalColumn.organization.name,
-                            AutoEvalColumn.knowledge_cutoff.name,
-
-                            AutoEvalColumn.score_math_algebra.name,
-                            AutoEvalColumn.score_math_geometry.name,
-                            AutoEvalColumn.score_math_probability.name,
-                            # AutoEvalColumn.rank_math_algebra.name,
-                            # AutoEvalColumn.rank_math_geometry.name,
-                            # AutoEvalColumn.rank_math_probability.name,
-                            ],
-                        rank_col=['sort_by_score'],
+            with gr.TabItem("🏆 Overview", elem_id="math_overview_subtab", id=0, elem_classes="subtab"): 
+                
+                with gr.TabItem("⭐ Sort by Rank", elem_id="math_overview_sort_by_rank_subtab", id=0, elem_classes="subtab"): 
+                    leaderboard = overall_leaderboard(
+                        get_model_leaderboard_df(
+                            model_result_path,
+                            benchmark_cols=[
+                                AutoEvalColumn.model.name, 
+                                # AutoEvalColumn.license.name,
+                                # AutoEvalColumn.organization.name,
+                                # AutoEvalColumn.knowledge_cutoff.name,
+
+                                # AutoEvalColumn.score_math_algebra.name,
+                                # AutoEvalColumn.score_math_geometry.name,
+                                # AutoEvalColumn.score_math_probability.name,
+                                AutoEvalColumn.rank_math_algebra.name,
+                                AutoEvalColumn.rank_math_geometry.name,
+                                AutoEvalColumn.rank_math_probability.name,
+                                ],
+                            rank_col=['sort_by_rank', 1, 4],
+                        )
                     )
-                )
+                    
+                with gr.TabItem("⭐ Sort by Score", elem_id="math_overview_sort_by_score_subtab", id=1, elem_classes="subtab"): 
+                    leaderboard = overall_leaderboard(
+                        get_model_leaderboard_df(
+                            model_result_path,
+                            benchmark_cols=[
+                                AutoEvalColumn.model.name, 
+                                # AutoEvalColumn.license.name,
+                                # AutoEvalColumn.organization.name,
+                                # AutoEvalColumn.knowledge_cutoff.name,
+
+                                AutoEvalColumn.score_math_algebra.name,
+                                AutoEvalColumn.score_math_geometry.name,
+                                AutoEvalColumn.score_math_probability.name,
+                                # AutoEvalColumn.rank_math_algebra.name,
+                                # AutoEvalColumn.rank_math_geometry.name,
+                                # AutoEvalColumn.rank_math_probability.name,
+                                ],
+                            rank_col=['sort_by_score', 1, 4],
+                        )
+                    )
+                    
 
 
             with gr.TabItem("🧮 Algebra", elem_id="algebra_subtab", id=1, elem_classes="subtab"): 
@@ -349,24 +374,42 @@ with demo:
             """
             gr.Markdown(DESCRIPTION_TEXT, elem_classes="markdown-text")
 
-            with gr.TabItem("Overall", elem_id="reasoning_overall_subtab", id=0, elem_classes="subtab"): 
-                leaderboard = overall_leaderboard(
-                    get_model_leaderboard_df(
-                        model_result_path,
-                        benchmark_cols=[
-                            AutoEvalColumn.model.name, 
-                            AutoEvalColumn.license.name,
-                            AutoEvalColumn.organization.name,
-                            AutoEvalColumn.knowledge_cutoff.name,
+            with gr.TabItem("🏆 Overview", elem_id="reasoning_overview_subtab", id=0, elem_classes="subtab"): 
+                
+                with gr.TabItem("⭐ Sort by Rank", elem_id="reasoning_overview_sort_by_rank_subtab", id=0, elem_classes="subtab"): 
+                    leaderboard = overall_leaderboard(
+                        get_model_leaderboard_df(
+                            model_result_path,
+                            benchmark_cols=[
+                                AutoEvalColumn.model.name, 
+                                # AutoEvalColumn.license.name,
+                                # AutoEvalColumn.organization.name,
+                                # AutoEvalColumn.knowledge_cutoff.name,
+
+                                AutoEvalColumn.rank_reason_logical.name,
+                                AutoEvalColumn.rank_reason_social.name,
+                                ],
+                            rank_col=['sort_by_rank', 1, 3],
+                        )
+                    )
 
-                            AutoEvalColumn.score_reason_logical.name,
-                            AutoEvalColumn.score_reason_social.name,
-                            # AutoEvalColumn.rank_reason_logical.name,
-                            # AutoEvalColumn.rank_reason_social.name,
-                            ],
-                        rank_col=['sort_by_score'],
+                with gr.TabItem("⭐ Sort by Score", elem_id="reasoning_overview_sort_by_score_subtab", id=1, elem_classes="subtab"): 
+                    leaderboard = overall_leaderboard(
+                        get_model_leaderboard_df(
+                            model_result_path,
+                            benchmark_cols=[
+                                AutoEvalColumn.model.name, 
+                                # AutoEvalColumn.license.name,
+                                # AutoEvalColumn.organization.name,
+                                # AutoEvalColumn.knowledge_cutoff.name,
+
+                                AutoEvalColumn.score_reason_logical.name,
+                                AutoEvalColumn.score_reason_social.name,
+                                ],
+                            rank_col=['sort_by_score', 1, 3],
+                        )
                     )
-                )
+
 
             with gr.TabItem("🧩 Logical", elem_id="logical_subtab", id=1, elem_classes="subtab"):         
                 leaderboard = overall_leaderboard(
@@ -430,23 +473,45 @@ with demo:
             """
             gr.Markdown(CURRENT_TEXT, elem_classes="markdown-text")
             
-            with gr.TabItem("Overall", elem_id="science_overall_subtab", id=0, elem_classes="subtab"): 
-                leaderboard = overall_leaderboard(
-                    get_model_leaderboard_df(
-                        model_result_path,
-                        benchmark_cols=[
-                            AutoEvalColumn.model.name, 
-                            AutoEvalColumn.license.name,
-                            AutoEvalColumn.organization.name,
-                            AutoEvalColumn.knowledge_cutoff.name,
-
-                            AutoEvalColumn.score_chemistry.name,
-                            # AutoEvalColumn.rank_chemistry.name,
-                            ],
-                        rank_col=['sort_by_score'],
+            with gr.TabItem("🏆 Overview", elem_id="science_overview_subtab", id=0, elem_classes="subtab"): 
+
+                with gr.TabItem("⭐ Sort by Rank", elem_id="science_overview_sort_by_rank_subtab", id=0, elem_classes="subtab"): 
+                    leaderboard = overall_leaderboard(
+                        get_model_leaderboard_df(
+                            model_result_path,
+                            benchmark_cols=[
+                                AutoEvalColumn.model.name, 
+                                
+                                AutoEvalColumn.license.name,
+                                AutoEvalColumn.organization.name,
+                                AutoEvalColumn.knowledge_cutoff.name,
+ 
+                                AutoEvalColumn.rank_chemistry.name,
+                                ],
+                            rank_col=['sort_by_rank', 4, 5],
+                        )
                     )
-                )
 
+                with gr.TabItem("⭐ Sort by Score", elem_id="science_overview_sort_by_score_subtab", id=1, elem_classes="subtab"): 
+                    leaderboard = overall_leaderboard(
+                        get_model_leaderboard_df(
+                            model_result_path,
+                            benchmark_cols=[
+                                AutoEvalColumn.model.name, 
+
+                                AutoEvalColumn.license.name,
+                                AutoEvalColumn.organization.name,
+                                AutoEvalColumn.knowledge_cutoff.name,
+
+                                AutoEvalColumn.score_chemistry.name,
+                                # AutoEvalColumn.rank_chemistry.name,
+                                ],
+                            rank_col=['sort_by_score', 4, 5], # two numbers are index to select the columns to average and sort
+                        )
+                    )
+                    
+                    
+                    
             with gr.TabItem("🧪 Chemistry", elem_id="chemistry_subtab", id=1, elem_classes="subtab"):         
                 leaderboard = overall_leaderboard(
                     get_model_leaderboard_df(

src/populate.py

@@ -15,7 +15,7 @@ def get_model_leaderboard_df(results_path: str, requests_path: str="", cols: lis
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_model_results(results_path)
     all_data_json = [v.to_dict() for v in raw_data]
-    assert len(rank_col) <= 1, "Only one column can be selected for ranking"
+    # assert len(rank_col) <= 1, "Only one column can be selected for ranking"
 
     df = pd.DataFrame.from_records(all_data_json)
     
@@ -24,7 +24,7 @@ def get_model_leaderboard_df(results_path: str, requests_path: str="", cols: lis
     
     # if there is one col in rank_col, this is an isolated dimension to rank by
     # sort by that selected column and remove NaN values
-    if rank_col and rank_col[0] != "sort_by_score": 
+    if rank_col and rank_col[0] not in ["sort_by_score", "sort_by_rank"]: 
         # df = df.dropna(subset=benchmark_cols)
         df = df.dropna(subset=rank_col)
         df = df.fillna(0.00)
@@ -39,23 +39,37 @@ def get_model_leaderboard_df(results_path: str, requests_path: str="", cols: lis
                 df[col] = df[col].round(decimals=2)
                 
     elif rank_col and rank_col[0] == "sort_by_score": # sorting by averaging all benchmark cols, except cols before offset_idx
-        offset_idx = 4
-        avg_scores = df.iloc[:, offset_idx:].mean(axis=1)
+        start_idx = rank_col[1]
+        end_idx = rank_col[2]
+        avg_scores = df.iloc[:, start_idx:end_idx].mean(axis=1)
         df.insert(1, "Average Score", avg_scores) 
         
         df["Average Score"] = avg_scores.round(decimals=4)
         df = df.sort_values(by=["Average Score"], ascending=False)
         df["Average Score"] = df["Average Score"].map('{:.2f}'.format)
         
-        df = df.drop(columns=benchmark_cols[offset_idx:])
+        # df = df.drop(columns=benchmark_cols[offset_idx:])
         # print(benchmark_cols)
         # print(df.head())
         # insert a rank column
         rank = np.arange(1, len(df)+1)
         df.insert(0, 'Rank', rank) 
-    
-    else:  # when rank_col, the first in benchmark_cols is empty, sort by averaging all the benchmarks, except the first one
-        avg_rank = df.iloc[:, 1:].mean(axis=1)
+
+        for col in benchmark_cols:
+            if 'Std dev' in col or 'Score' in col:
+                df[col] = (df[col]).map('{:.2f}'.format)
+                df[col] = df[col].round(decimals=2)
+
+        # df = df.fillna('--')
+        df.replace("nan", '--', inplace=True)
+        
+    elif rank_col and rank_col[0] == "sort_by_rank":
+    # else:  # when rank_col, the first in benchmark_cols is empty, sort by averaging all the benchmarks, except the first one
+        start_idx = rank_col[1]
+        end_idx = rank_col[2]
+        avg_rank = df.iloc[:, start_idx:end_idx].mean(axis=1)
+        df.insert(1, "Average Rank", avg_rank) 
+        
         df["Average Rank"] = avg_rank.round(decimals=4)
         df = df.sort_values(by=["Average Rank"], ascending=True)        
         df["Average Rank"] = df["Average Rank"].map('{:.2f}'.format)
@@ -66,6 +80,8 @@ def get_model_leaderboard_df(results_path: str, requests_path: str="", cols: lis
         rank = np.arange(1, len(df)+1)
         df.insert(0, 'Rank', rank)
         
+        # df.style.background_gradient(cmap='coolwarm', subset=benchmark_cols)
+