Spaces:
Running
Running
| import gradio as gr | |
| import pandas as pd | |
| import plotly.express as px | |
| import os # To check if files exist | |
| # --- Configuration --- | |
| DATA_DIR = "." # Assume CSV files are in the same directory as app.py | |
| SUMMARY_FILE = os.path.join(DATA_DIR, "data/summary_data.csv") | |
| DOMAIN_RANKS_FILE = os.path.join(DATA_DIR, "data/domain_ranks.csv") | |
| COST_FILE = os.path.join(DATA_DIR, "data/cost_data.csv") | |
| AVG_LATENCY_FILE = os.path.join(DATA_DIR, "data/avg_latency.csv") | |
| P99_LATENCY_FILE = os.path.join(DATA_DIR, "data/p99_latency.csv") | |
| # --- Helper Function to Load Data --- | |
| def load_data(filepath, separator=','): | |
| """Loads data, handling potential file not found errors.""" | |
| if not os.path.exists(filepath): | |
| print(f"Warning: Data file not found at {filepath}") | |
| return pd.DataFrame() # Return empty DataFrame | |
| try: | |
| # Adjust separator if needed (e.g., sep='\t' for tab-separated) | |
| df = pd.read_csv(filepath, sep=separator) | |
| # Basic cleanup: remove potential unnamed index columns often added by spreadsheets | |
| df = df.loc[:, ~df.columns.str.contains('^Unnamed')] | |
| # Attempt to convert numeric columns, coercing errors to NaN | |
| for col in df.columns: | |
| if col != 'Model Name' and col != 'model_name': # Avoid converting model names | |
| # Check if column might represent numeric data before converting | |
| if df[col].astype(str).str.contains(r'^[0-9.,eE-]+$').any(): | |
| df[col] = pd.to_numeric(df[col], errors='coerce') | |
| return df | |
| except Exception as e: | |
| print(f"Error loading {filepath}: {e}") | |
| return pd.DataFrame() | |
| # --- Load All Data --- | |
| print("Loading data...") | |
| df_summary = load_data(SUMMARY_FILE) | |
| df_domain = load_data(DOMAIN_RANKS_FILE) | |
| df_cost = load_data(COST_FILE) | |
| df_avg_latency = load_data(AVG_LATENCY_FILE) | |
| df_p99_latency = load_data(P99_LATENCY_FILE) | |
| print("Data loading complete.") | |
| # --- *** NEW: Convert Costs to USD Cents *** --- | |
| COST_COLUMN_SUMMARY = 'Costs (USD)' # IMPORTANT: Check this matches your summary_data.csv header EXACTLY | |
| NEW_COST_COLUMN_SUMMARY = 'Avg Cost ($ Cents)' # This is the new name we'll use | |
| # Convert summary cost | |
| if not df_summary.empty and COST_COLUMN_SUMMARY in df_summary.columns: | |
| df_summary[COST_COLUMN_SUMMARY] = (pd.to_numeric(df_summary[COST_COLUMN_SUMMARY], errors='coerce') * 100).round(3) # <-- ADDED .round(3) | |
| df_summary.rename(columns={COST_COLUMN_SUMMARY: NEW_COST_COLUMN_SUMMARY}, inplace=True) | |
| print(f"Converted '{COST_COLUMN_SUMMARY}' to $ Cents and renamed to '{NEW_COST_COLUMN_SUMMARY}' in df_summary.") | |
| else: | |
| print(f"Warning: Column '{COST_COLUMN_SUMMARY}' not found in df_summary for conversion.") | |
| # Convert cost breakdown data | |
| if not df_cost.empty: | |
| # IMPORTANT: Check if your model name column in cost_data.csv is 'model_name' or 'Model Name' etc. | |
| model_col_name = 'model_name' # Adjust if needed | |
| cost_cols = [col for col in df_cost.columns if col != model_col_name] | |
| for col in cost_cols: | |
| # Handle potential non-numeric data gracefully before multiplying | |
| df_cost[col] = (pd.to_numeric(df_cost[col], errors='coerce') * 100).round(3) # <-- ADDED .round(3) | |
| print("Converted cost breakdown columns to $ Cents in df_cost.") | |
| # --- *** End of Cost Conversion *** --- | |
| # Rename columns for clarity if needed (example for summary) | |
| # Make sure the original names match your CSV headers EXACTLY | |
| try: | |
| df_summary = df_summary.rename(columns={ | |
| 'Model Name': 'Model', # If your CSV uses 'Model Name' | |
| # Add other renames here if your CSV headers differ from the target names below | |
| # 'Costs (USD)': 'Avg Cost (USD/response)', | |
| # 'Avg Answer Duration (sec)': 'Avg Latency (s)', | |
| # 'P99 Answer Duration (sec)': 'P99 Latency (s)' | |
| }) | |
| # Select and reorder columns for the main table - REMOVED BENCHMARK COLUMNS | |
| summary_cols_display = ['Model', 'AutoBench', NEW_COST_COLUMN_SUMMARY, 'Avg Answer Duration (sec)', 'P99 Answer Duration (sec)'] | |
| # Filter to only columns that actually exist after loading and renaming | |
| summary_cols_display = [col for col in summary_cols_display if col in df_summary.columns] | |
| df_summary_display = df_summary[summary_cols_display].copy() # Use .copy() to avoid SettingWithCopyWarning | |
| # Select columns for the new benchmark comparison table | |
| benchmark_cols = ['Model', 'AutoBench', 'Chatbot Ar.', 'AAI Index', 'MMLU Index'] | |
| benchmark_cols = [col for col in benchmark_cols if col in df_summary.columns] # Filter existing | |
| df_benchmark_display = df_summary[benchmark_cols].copy() # Use .copy() | |
| # Ensure AutoBench score is numeric for sorting BOTH display tables | |
| if 'AutoBench' in df_summary_display.columns: | |
| df_summary_display['AutoBench'] = pd.to_numeric(df_summary_display['AutoBench'], errors='coerce') | |
| df_summary_display.sort_values(by='AutoBench', ascending=False, inplace=True) # Use inplace=True | |
| else: | |
| print("Warning: 'AutoBench' column not found for sorting summary table.") | |
| if 'AutoBench' in df_benchmark_display.columns: | |
| df_benchmark_display['AutoBench'] = pd.to_numeric(df_benchmark_display['AutoBench'], errors='coerce') | |
| df_benchmark_display.sort_values(by='AutoBench', ascending=False, inplace=True) # Use inplace=True | |
| else: | |
| print("Warning: 'AutoBench' column not found for sorting benchmark table.") | |
| except KeyError as e: | |
| print(f"Error preparing display columns: Missing key {e}. Check CSV headers and rename mapping.") | |
| df_summary_display = df_summary.copy() # Fallback | |
| df_benchmark_display = pd.DataFrame() # Fallback to empty for benchmark table | |
| # --- Build Gradio App --- | |
| with gr.Blocks(theme=gr.themes.Soft()) as app: | |
| gr.Markdown("# AutoBench LLM Leaderboard") | |
| gr.Markdown( | |
| "Interactive leaderboard for AutoBench, where LLMs rank LLMs' responses. " | |
| "Includes performance, cost, and latency metrics." | |
| "Data updated on April 25, 2025." | |
| "\n\nMore info for this benchmark run: [AutoBench Run 2 Results](https://huggingface.co/blog/PeterKruger/autobench-2nd-run). " | |
| "If you want to know more about AutoBench: [AutoBench Release](https://huggingface.co/blog/PeterKruger/autobench)." | |
| ) | |
| # --- Tab 1: Overall Ranking --- | |
| with gr.Tab("Overall Ranking"): | |
| gr.Markdown("## Overall Model Performance") | |
| # REMOVED benchmark correlations from Markdown | |
| gr.Markdown("Models ranked by AutoBench score. Lower cost ($ Cents) and latency (s) are better.") | |
| # Check if df_summary_display has data before rendering | |
| if not df_summary_display.empty: | |
| # Create a copy specifically for this tab's display and rename the column | |
| df_overall_rank_display = df_summary_display.copy() | |
| if 'AutoBench' in df_overall_rank_display.columns: | |
| df_overall_rank_display.rename(columns={'AutoBench': 'Rank'}, inplace=True) | |
| gr.DataFrame( | |
| df_overall_rank_display, # Pass the renamed DF | |
| # Adjust datatype length based on potentially fewer columns | |
| datatype=['str'] + ['number'] * (len(df_overall_rank_display.columns) - 1), | |
| interactive=True, # Allows sorting | |
| # height=600 # Adjust height as needed | |
| ) | |
| else: | |
| gr.Markdown("_(Summary data failed to load or is empty. Please check `summary_data.csv`)_") | |
| # --- NEW Tab 1.5: Benchmark Comparison --- | |
| with gr.Tab("Benchmark Comparison"): | |
| gr.Markdown("## Benchmark Comparison") | |
| gr.Markdown("Comparison of AutoBench scores with other popular benchmarks. AutoBench features 82.51% correlation with Chatbot Arena, 83.74% with Artificial Analysis Intelligence Index, and 71.51% with MMLU. Models sorted by AutoBench score.") | |
| if not df_benchmark_display.empty: | |
| gr.DataFrame( | |
| df_benchmark_display, | |
| datatype=['str'] + ['number'] * (len(df_benchmark_display.columns) - 1), | |
| interactive=True # Allow sorting | |
| ) | |
| else: | |
| gr.Markdown("_(Benchmark comparison data could not be prepared. Check `summary_data.csv` for 'Chatbot Ar.', 'AAI Index', 'MMLU Index' columns.)_") | |
| # --- Tab 2: Performance Plots --- | |
| with gr.Tab("Performance Plots"): | |
| gr.Markdown("## Performance Visualizations") | |
| gr.Markdown("Exploring relationships between AutoBench Rank, Latency, and Cost.") | |
| # Scatter Plot 1 (using summary data) | |
| gr.Markdown("### Rank vs. Average Cost") | |
| if not df_summary.empty and 'AutoBench' in df_summary.columns and NEW_COST_COLUMN_SUMMARY in df_summary.columns: | |
| # Filter out rows where essential plot data might be missing | |
| plot_df = df_summary.dropna(subset=['AutoBench', NEW_COST_COLUMN_SUMMARY, 'Model']).copy() | |
| plot_df[NEW_COST_COLUMN_SUMMARY] = pd.to_numeric(plot_df[NEW_COST_COLUMN_SUMMARY], errors='coerce') | |
| plot_df = plot_df.dropna(subset=[NEW_COST_COLUMN_SUMMARY]) # Drop if cost conversion failed | |
| if not plot_df.empty: | |
| fig_cost = px.scatter( | |
| plot_df, | |
| x=NEW_COST_COLUMN_SUMMARY, | |
| y="AutoBench", | |
| text="Model", # Show model name near point | |
| log_x=True, # Use log scale for cost | |
| title="AutoBench Rank vs. Average Cost per Response ($ Cents - Log Scale)", | |
| labels={'AutoBench': 'AutoBench Rank', NEW_COST_COLUMN_SUMMARY: 'Avg Cost ($ Cents) - Log Scale'}, | |
| hover_data=['Model', 'AutoBench', NEW_COST_COLUMN_SUMMARY, 'Avg Answer Duration (sec)'] # Show details on hover | |
| ) | |
| fig_cost.update_traces(textposition='top center') | |
| fig_cost.update_layout( | |
| xaxis_title="Avg Cost ($ Cents) - Log Scale", # Keep bottom axis title | |
| yaxis_title="AutoBench Rank", | |
| width=1000, # Your existing width | |
| height=800, # Your existing height (if you added it) | |
| # --- ADD THE FOLLOWING --- | |
| xaxis2=dict( | |
| overlaying='x', # Link to primary x-axis | |
| matches='x', # Explicitly match primary x-axis properties (like type='log') | |
| side='top', # Position on top | |
| showticklabels=True,# Show the labels (numbers) | |
| showline=True, # Explicitly show the axis line itself | |
| title=None # No title for the top axis | |
| ) | |
| # --- END OF ADDITION --- | |
| ) | |
| gr.Plot(fig_cost) | |
| else: | |
| gr.Markdown("_(Insufficient valid data for Rank vs Cost plot. Check 'AutoBench' and NEW_COST_COLUMN_SUMMARY columns in `summary_data.csv`)_") | |
| else: | |
| gr.Markdown("_(Summary data failed to load or essential columns missing for Rank vs Cost plot)_") | |
| # Plot 2: Rank vs Average Latency | |
| gr.Markdown("### Rank vs. Average Latency") | |
| if not df_summary.empty and 'AutoBench' in df_summary.columns and 'Avg Answer Duration (sec)' in df_summary.columns: | |
| # Filter out rows where essential plot data might be missing | |
| plot_df_avg_latency = df_summary.dropna(subset=['AutoBench', 'Avg Answer Duration (sec)', 'Model']).copy() | |
| plot_df_avg_latency['Avg Answer Duration (sec)'] = pd.to_numeric(plot_df_avg_latency['Avg Answer Duration (sec)'], errors='coerce') | |
| plot_df_avg_latency = plot_df_avg_latency.dropna(subset=['Avg Answer Duration (sec)']) # Drop if conversion failed | |
| if not plot_df_avg_latency.empty: | |
| fig_avg_latency = px.scatter( | |
| plot_df_avg_latency, | |
| x="Avg Answer Duration (sec)", | |
| y="AutoBench", | |
| text="Model", | |
| log_x=True, # Use log scale for latency - adjust if not desired | |
| title="AutoBench Rank vs. Average Latency (Log Scale)", | |
| labels={'AutoBench': 'AutoBench Rank', 'Avg Answer Duration (sec)': 'Avg Latency (s) - Log Scale'}, | |
| hover_data=['Model', 'AutoBench', 'Avg Answer Duration (sec)', NEW_COST_COLUMN_SUMMARY] | |
| ) | |
| fig_avg_latency.update_traces(textposition='top center') | |
| fig_avg_latency.update_layout(xaxis_title="Avg Latency (s) - Log Scale", yaxis_title="AutoBench Rank", width=1000, height=800) | |
| gr.Plot(fig_avg_latency) | |
| else: | |
| gr.Markdown("_(Insufficient valid data for Rank vs Avg Latency plot. Check 'AutoBench' and 'Avg Answer Duration (sec)' columns in `summary_data.csv`)_") | |
| else: | |
| gr.Markdown("_(Summary data failed to load or essential columns missing for Rank vs Avg Latency plot)_") | |
| # Plot 3: Rank vs P99 Latency | |
| gr.Markdown("### Rank vs. P99 Latency") | |
| if not df_summary.empty and 'AutoBench' in df_summary.columns and 'P99 Answer Duration (sec)' in df_summary.columns: | |
| # Filter out rows where essential plot data might be missing | |
| plot_df_p99_latency = df_summary.dropna(subset=['AutoBench', 'P99 Answer Duration (sec)', 'Model']).copy() | |
| plot_df_p99_latency['P99 Answer Duration (sec)'] = pd.to_numeric(plot_df_p99_latency['P99 Answer Duration (sec)'], errors='coerce') | |
| plot_df_p99_latency = plot_df_p99_latency.dropna(subset=['P99 Answer Duration (sec)']) # Drop if conversion failed | |
| if not plot_df_p99_latency.empty: | |
| fig_p99_latency = px.scatter( | |
| plot_df_p99_latency, | |
| x="P99 Answer Duration (sec)", | |
| y="AutoBench", | |
| text="Model", | |
| log_x=True, # Use log scale for latency - adjust if not desired | |
| title="AutoBench Rank vs. P99 Latency (Log Scale)", | |
| labels={'AutoBench': 'AutoBench Rank', 'P99 Answer Duration (sec)': 'P99 Latency (s) - Log Scale'}, | |
| hover_data=['Model', 'AutoBench', 'P99 Answer Duration (sec)', 'Avg Answer Duration (sec)', NEW_COST_COLUMN_SUMMARY] | |
| ) | |
| fig_p99_latency.update_traces(textposition='top center') | |
| fig_p99_latency.update_layout(xaxis_title="P99 Latency (s) - Log Scale", yaxis_title="AutoBench Rank", width=1000, height=800) | |
| gr.Plot(fig_p99_latency) | |
| else: | |
| gr.Markdown("_(Insufficient valid data for Rank vs P99 Latency plot. Check 'AutoBench' and 'P99 Answer Duration (sec)' columns in `summary_data.csv`)_") | |
| else: | |
| gr.Markdown("_(Summary data failed to load or essential columns missing for Rank vs P99 Latency plot)_") | |
| # --- Tab 3: Cost & Latency Analysis --- | |
| with gr.Tab("Cost & Latency Analysis"): | |
| gr.Markdown("## Performance vs. Cost/Latency Trade-offs") | |
| # Cost Breakdown Table | |
| gr.Markdown("### Cost Breakdown per Domain ($ Cents/Response)") # <-- MODIFIED | |
| if not df_cost.empty: | |
| # Make model name the first column if it exists | |
| if 'model_name' in df_cost.columns: | |
| cols = ['model_name'] + [col for col in df_cost.columns if col != 'model_name'] | |
| df_cost_display = df_cost[cols] | |
| else: | |
| df_cost_display = df_cost # Use as is if 'model_name' isn't found | |
| gr.DataFrame(df_cost_display, interactive=True) | |
| else: | |
| gr.Markdown("_(Cost breakdown data failed to load or is empty. Please check `cost_data.csv`)_") | |
| # Latency Breakdown Tables | |
| gr.Markdown("### Average Latency Breakdown per Domain (Seconds)") | |
| if not df_avg_latency.empty: | |
| if 'model_name' in df_avg_latency.columns: | |
| cols = ['model_name'] + [col for col in df_avg_latency.columns if col != 'model_name'] | |
| df_avg_latency_display = df_avg_latency[cols] | |
| else: | |
| df_avg_latency_display = df_avg_latency | |
| gr.DataFrame(df_avg_latency_display, interactive=True) | |
| else: | |
| gr.Markdown("_(Average latency data failed to load or is empty. Please check `avg_latency.csv`)_") | |
| gr.Markdown("### P99 Latency Breakdown per Domain (Seconds)") | |
| if not df_p99_latency.empty: | |
| if 'model_name' in df_p99_latency.columns: | |
| cols = ['model_name'] + [col for col in df_p99_latency.columns if col != 'model_name'] | |
| df_p99_latency_display = df_p99_latency[cols] | |
| else: | |
| df_p99_latency_display = df_p99_latency | |
| gr.DataFrame(df_p99_latency_display, interactive=True) | |
| else: | |
| gr.Markdown("_(P99 latency data failed to load or is empty. Please check `p99_latency.csv`)_") | |
| # --- Tab 4: Domain Performance --- | |
| with gr.Tab("Domain Performance"): | |
| gr.Markdown("## Performance Across Different Domains") | |
| gr.Markdown("Model ranks within specific knowledge or task areas. Higher is better.") | |
| if not df_domain.empty: | |
| if 'Model Name' in df_domain.columns: | |
| # Attempt to make Model Name first col | |
| cols = ['Model Name'] + [col for col in df_domain.columns if col != 'Model Name'] | |
| df_domain_display = df_domain[cols] | |
| else: | |
| df_domain_display = df_domain # Use as is | |
| gr.DataFrame(df_domain_display, interactive=True) | |
| else: | |
| gr.Markdown("_(Domain ranks data failed to load or is empty. Please check `domain_ranks.csv`)_") | |
| # --- Tab 5: About --- | |
| with gr.Tab("About AutoBench"): | |
| gr.Markdown(""" | |
| ## About AutoBench | |
| AutoBench is an LLM benchmark where Large Language Models (LLMs) evaluate and rank the responses generated by other LLMs. The questions themselves are also generated by LLMs across a diverse set of domains and ranked for quality. | |
| ### Methodology | |
| 1. **Question Generation:** High-quality questions across various domains (Coding, History, Science, etc.) are generated by selected LLMs. | |
| 2. **Response Generation:** The models being benchmarked generate answers to these questions. | |
| 3. **Ranking:** Ranking LLMs rank the responses from different models for each question, on a 1-5 scale. | |
| 4. **Aggregation:** Scores are averaged across multiple questions and domains to produce the final AutoBench rank. | |
| ### Metrics | |
| * **AutoBench Score (AB):** The average rank received by a model's responses across all questions/domains (higher is better). | |
| * **Avg Cost (USD Cents/response):** Estimated average cost to generate one response based on model provider pricing (input+output tokens). Lower is better. | |
| * **Avg Latency (s):** Average time taken by the model to generate a response. Lower is better. | |
| * **P99 Latency (s):** The 99th percentile of response time, indicating worst-case latency. Lower is better. | |
| * **Chatbot Arena / Artificial Analysis Intelligence Index / MMLU:** Scores from other well-known benchmarks for comparison (where available). | |
| ### Data | |
| This leaderboard reflects a run completed on April 23, 2025. Models included recently released models such as o4-mini, Gpt-4.1-mini, Gemini 2.5 Pro Preview, Claude 3.7 Sonnet:thikning, etc.. | |
| ### Links | |
| * [AutoBench Run 2 Results](https://huggingface.co/blog/PeterKruger/autobench-2nd-run) | |
| * [AutoBench Blog Post](https://huggingface.co/blog/PeterKruger/autobench) | |
| * [Autobench Repositories](https://huggingface.co/AutoBench) | |
| **Disclaimer:** Benchmark results provide one perspective on model capabilities. Performance can vary based on specific tasks, prompts, and API conditions. Costs are estimates and subject to change by providers. Latency depends on server load and geographic location. | |
| """) | |
| # --- Launch the App --- | |
| print("Launching Gradio app...") | |
| app.launch() | |
| print("Gradio app launched.") |