edit

app.py

@@ -5,6 +5,8 @@ os.environ['TMP_DIR'] = "tmp"
 import subprocess
 import shutil
 import gradio as gr
+import numpy as np
+from src.radial.radial import create_plot
 from apscheduler.schedulers.background import BackgroundScheduler
 from gradio_leaderboard import Leaderboard, SelectColumns
 from gradio_space_ci import enable_space_ci
@@ -17,7 +19,6 @@ def handle_file_upload(file):
     with open(file.name, "r") as f:
         v = json.load(f)
     return v, file_path
-
 def submit_file(v, file_path, mn, profile: gr.OAuthProfile | None):
     if profile is None:
         return "Hub Login Required"
@@ -67,16 +68,19 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(
 # Start ephemeral Spaces on PRs (see config in README.md)
 enable_space_ci()
 
-download_openbench()
+# download_openbench()
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
     download_openbench()
 
+def update_plot(selected_models):
+    return create_plot(selected_models)
 
 def build_demo():
     demo = gr.Blocks(title="Small Shlepa", css=custom_css)
     leaderboard_df = build_leadearboard_df()
+    default_model = leaderboard_df["model"].iloc[0]
     with demo:
         gr.HTML(TITLE)
         gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
@@ -167,11 +171,26 @@ def build_demo():
                         [out]
                     )
 
-            with gr.TabItem("Analytics", elem_id="llm-benchmark-tab-table", id=4):
-                with gr.Column():
-                   
-                    model_name_textbox = gr.Textbox(label="Model name")
-
+            with gr.TabItem("📊 Analytics", elem_id="llm-benchmark-tab-table", id=4):
+                    with gr.Column():
+                        model_dropdown = gr.Dropdown(
+                            choices=leaderboard_df["model"].tolist(), 
+                            label="Models", 
+                            value=default_model,
+                            multiselect=True,
+                            info="Select models"
+                        )
+                    with gr.Column():
+                        plot = gr.Plot(value=
+                            update_plot(
+                                default_model,
+                            )
+                        )
+                    model_dropdown.change(
+                            fn=update_plot,
+                            inputs=[model_dropdown],
+                            outputs=[plot]
+                    )
                     return demo
 
 
@@ -227,7 +246,7 @@ if __name__ == "__main__":
 
     scheduler = BackgroundScheduler()
     # update_board()
-    scheduler.add_job(update_board, "interval", minutes=600)
+    # scheduler.add_job(update_board, "interval", minutes=600)
     scheduler.start()
 
     demo_app = build_demo()

data/leaderboard.json

@@ -0,0 +1,42 @@
+[
+    {
+        "musicmc": 0.09361702127659574,
+        "mmluproru": 0.10207253886010363,
+        "lawmc": 0.11431513903192585,
+        "model": "NousResearch/Llama-2-7b-hf",
+        "moviesmc": 0.07175925925925926,
+        "booksmc": 0.1078838174273859,
+        "model_dtype": "torch.float16",
+        "ppl": 0
+    },
+    {
+        "musicmc": 0.2553191489361702,
+        "mmluproru": 0.2621761658031088,
+        "lawmc": 0.5818743563336766,
+        "model": "google/gemma-2-9b",
+        "moviesmc": 0.5046296296296297,
+        "booksmc": 0.3360995850622407,
+        "model_dtype": "torch.float16",
+        "ppl": 0
+    },
+    {
+        "musicmc": 0.2680851063829787,
+        "mmluproru": 0.20103626943005182,
+        "lawmc": 0.5386199794026777,
+        "model": "Vikhrmodels/it-5.2-fp16-cp",
+        "moviesmc": 0.4537037037037037,
+        "booksmc": 0.3070539419087137,
+        "model_dtype": "torch.float16",
+        "ppl": 0
+    },
+    {
+        "musicmc": 0.25957446808510637,
+        "mmluproru": 0.19378238341968912,
+        "lawmc": 0.518022657054583,
+        "model": "lightblue/suzume-llama-3-8B-multilingual",
+        "moviesmc": 0.3287037037037037,
+        "booksmc": 0.2966804979253112,
+        "model_dtype": "torch.float16",
+        "ppl": 0
+    }
+]

m_data/generate_leaderboard.py

@@ -0,0 +1,29 @@
+import os
+import json
+
+def merge_json_files(folder_path, output_file):
+    merged_data = []
+    if not os.path.exists(folder_path):
+        print(f"Папка '{folder_path}' не существует.")
+        return
+    
+    for filename in os.listdir(folder_path):
+        if filename.endswith('.json'):
+            file_path = os.path.join(folder_path, filename)
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    merged_data.append(data)
+            except Exception as e:
+                print(f"Ошибка при чтении файла '{file_path}': {e}")
+    
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(merged_data, f, ensure_ascii=False, indent=4)
+    
+    print(f"Данные успешно объединены и сохранены в файл '{output_file}'.")
+
+if __name__ == "__main__":
+    folder_path = './model_data/external'  
+    output_file = 'leaderboard.json'
+    
+    merge_json_files(folder_path, output_file)

src/radial/radial.py

@@ -0,0 +1,67 @@
+import plotly.graph_objects as go
+import random
+from src.leaderboard.build_leaderboard import build_leadearboard_df
+import numpy as np
+import itertools as it
+
+
+def create_plot(selected_models):
+    models = build_leadearboard_df()
+    metrics = ["musicmc", "lawmc", "moviesmc", "booksmc", "mmluproru"]
+    MIN_COLOUR_DISTANCE_BETWEEN_MODELS = 100
+    seed = 42
+    
+    def generate_colours(min_distance, seed):
+        colour_mapping = {}
+        all_models = selected_models
+        
+        for i in it.count():
+            min_colour_distance = min_distance - i
+            retries_left = 10 * len(all_models)
+            
+            for model_id in all_models:
+                random.seed(hash(model_id) + i + seed)
+                r, g, b = 0, 0, 0
+                too_bright, similar_to_other_model = True, True
+                
+                while (too_bright or similar_to_other_model) and retries_left > 0:
+                    r, g, b = tuple(random.randint(0, 255) for _ in range(3))
+                    too_bright = np.min([r, g, b]) > 200
+                    similar_to_other_model = any(
+                        np.abs(np.array(colour) - np.array([r, g, b])).sum() < min_colour_distance
+                        for colour in colour_mapping.values()
+                    )
+                    retries_left -= 1
+                
+                colour_mapping[model_id] = (r, g, b)
+            if len(colour_mapping) == len(all_models):
+                break
+        
+        return colour_mapping
+
+    colour_mapping = generate_colours(MIN_COLOUR_DISTANCE_BETWEEN_MODELS, seed)
+    fig = go.Figure()
+    for _, model_data in models.iterrows():
+        model_name = model_data["model"]
+        if not model_name in selected_models:
+            continue
+        values = [model_data[metric] for metric in metrics]
+        color = f'rgb{colour_mapping[model_name]}'
+
+        fig.add_trace(go.Scatterpolar(
+            r=values,
+            theta=metrics,
+            name=model_name,
+            fill='toself',
+            fillcolor=f'rgba{colour_mapping[model_name] + (0.6,)}',
+            line=dict(color=color)
+        ))
+    fig.update_layout(
+        polar=dict(radialaxis=dict(visible=True)),
+        showlegend=True,
+        title='Models metrics',
+        template="plotly_dark",
+    )
+    return fig
+
+