initial commit

.gitignore

@@ -0,0 +1,2 @@
+scripts/
+__pycache__/

README.md

@@ -0,0 +1,77 @@
+---
+title: Physical AI Bench Leaderboard
+emoji: 🤖
+colorFrom: blue
+colorTo: purple
+sdk: gradio
+app_file: app.py
+pinned: true
+license: mit
+short_description: Benchmark for Physical AI generation and understanding
+sdk_version: 5.43.1
+tags:
+- leaderboard
+- physical-ai
+- world-models
+- autonomous-driving
+- robotics
+- embodied-ai
+---
+
+# Physical AI Bench Leaderboard
+
+**Physical AI Bench (PAI-Bench)** is a comprehensive benchmark suite for evaluating physical AI generation and understanding across diverse scenarios including autonomous vehicles, robotics, industrial spaces, and everyday ego-centric environments.
+
+## Resources
+
+- 🌐 [GitHub Repository](https://github.com/SHI-Labs/physical-ai-bench)
+- 📊 [Predict Dataset](https://huggingface.co/datasets/shi-labs/physical-ai-bench-predict)
+- 📊 [Transfer Dataset](https://huggingface.co/datasets/shi-labs/physical-ai-bench-transfer)
+- 📊 [Reason Dataset](https://huggingface.co/datasets/shi-labs/physical-ai-bench-reason)
+
+## Citation
+
+```bibtex
+@misc{PAIBench2025,
+  title={Physical AI Bench: A Comprehensive Benchmark for Physical AI Generation and Understanding},
+  author={Fengzhe Zhou and Jiannan Huang and Jialuo Li and Humphrey Shi},
+  year={2025},
+  url={https://github.com/SHI-Labs/physical-ai-bench}
+}
+```
+
+---
+
+# Configuration
+
+Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
+
+Results files should have the following format and be stored as json files:
+```json
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.
+
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+
+# Code logic for more complex edits
+
+You'll find
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -0,0 +1,617 @@
+import gradio as gr
+import pandas as pd
+
+
+# Your leaderboard name
+TITLE = """<h1 align="center" id="space-title">Physical AI Bench Leaderboard</h1>"""
+
+# What does your leaderboard evaluate?
+INTRODUCTION_TEXT = """
+**Physical AI Bench (PAI-Bench)** is a comprehensive benchmark suite for evaluating physical AI generation and understanding across diverse scenarios including autonomous vehicles, robotics, industrial spaces, and everyday ego-centric environments.
+"""
+
+# Which evaluations are you running? how can people reproduce what you have?
+LLM_BENCHMARKS_TEXT = """
+## How it works
+
+This leaderboard tracks model performance across three core dimensions:
+
+- **🎨 Predict**: Evaluates world foundation models' ability to predict future states across 1,044 diverse physical scenarios
+- **🔄 Transfer**: Focuses on world model generation with complex control signals, featuring 600 videos across robotic arm operations, autonomous driving, and ego-centric scenes
+- **🧠 Reason**: Evaluates understanding and reasoning about physical scenes, with 1,214 embodied reasoning scenarios focused on autonomous vehicle actions
+
+PAI-Bench covers multiple physical AI domains including autonomous driving, robotics, industrial spaces, physics simulations, human interactions, and common sense reasoning.
+
+### Resources
+- 🌐 [GitHub Repository](https://github.com/SHI-Labs/physical-ai-bench)
+- 📊 [Predict Dataset](https://huggingface.co/datasets/shi-labs/physical-ai-bench-predict)
+- 📊 [Transfer Dataset](https://huggingface.co/datasets/shi-labs/physical-ai-bench-transfer)
+- 📊 [Reason Dataset](https://huggingface.co/datasets/shi-labs/physical-ai-bench-reason)
+
+## Reproducibility
+
+To evaluate your models on PAI-Bench, visit our [GitHub repository](https://github.com/SHI-Labs/physical-ai-bench) for evaluation scripts and detailed instructions.
+
+## Citation
+
+If you use Physical AI Bench in your research, please cite:
+
+```bibtex
+@misc{{PAIBench2025,
+  title={{Physical AI Bench: A Comprehensive Benchmark for Physical AI Generation and Understanding}},
+  author={{Fengzhe Zhou and Jiannan Huang and Jialuo Li and Humphrey Shi}},
+  year={{2025}},
+  url={{https://github.com/SHI-Labs/physical-ai-bench}}
+}}
+```
+"""
+
+
+# ============================================================================
+# Model Links Utility
+# ============================================================================
+
+def create_model_link(model_name):
+    """
+    Convert a model name to a markdown link to Hugging Face.
+
+    Args:
+        model_name: Model name in format "org/model-name" or just a plain name
+
+    Returns:
+        Markdown formatted link or original name if format doesn't match
+    """
+    if not isinstance(model_name, str):
+        return model_name
+
+    # Check if the model name follows the "org/model" format
+    if '/' in model_name:
+        # This is likely a HuggingFace model ID
+        hf_url = f"https://huggingface.co/{model_name}"
+        return f"[{model_name}]({hf_url})"
+
+    # If it doesn't have a slash, return as-is
+    return model_name
+
+
+# ============================================================================
+# Predict Tab Configuration and Utilities
+# ============================================================================
+
+# Column name mapping (from original name to display name)
+PREDICT_COLUMN_NAME_MAPPING = {
+    'Common+Misc': 'Common Sense',
+    'BG Consistency': 'Background Consistency',
+    'Motion': 'Motion Smoothness',
+    'Aesthetic': 'Aesthetic Quality',
+    'I2V BG': 'I2V Background'
+}
+
+# Columns to remove from the dataframe
+PREDICT_COLUMNS_TO_REMOVE = ['Avg Score/Video', 'Common', 'Misc']
+
+# Desired column order (using renamed column names)
+PREDICT_COLUMN_ORDER = [
+    'model',
+    'Overall',
+    'Domain Score',
+    'Quality Score',
+    'Common Sense',
+    'AV',
+    'Robot',
+    'Industry',
+    'Human',
+    'Physics',
+    'Subject Consistency',
+    'Background Consistency',
+    'Motion Smoothness',
+    'Aesthetic Quality',
+    'Image Quality',
+    'Overall Consistency',
+    'I2V Subject',
+    'I2V Background',
+    'params',
+    'activate_params'
+]
+
+# Columns to hide by default (but still available for filtering/selection)
+PREDICT_HIDDEN_COLUMNS = ['params', 'activate_params']
+
+# Semantic/Domain dimensions (for selection button)
+PREDICT_DOMAIN_SCORE_DIMENSIONS = [
+    'Domain Score',
+    'Common Sense',
+    'AV',
+    'Robot',
+    'Industry',
+    'Human',
+    'Physics',
+]
+
+# Quality dimensions (for selection button)
+PREDICT_QUALITY_SCORE_DIMENSIONS = [
+    'Quality Score',
+    'Subject Consistency',
+    'Background Consistency',
+    'Motion Smoothness',
+    'Aesthetic Quality',
+    'Image Quality',
+    'Overall Consistency',
+    'I2V Subject',
+    'I2V Background'
+]
+
+PREDICT_DESELECTED_COLUMNS = ['Domain Score', 'Quality Score']
+
+PREDICT_ALL_SELECTED_COLUMNS = [
+    'Domain Score',
+    'Quality Score',
+    'Common Sense',
+    'AV',
+    'Robot',
+    'Industry',
+    'Human',
+    'Physics',
+    'Subject Consistency',
+    'Background Consistency',
+    'Motion Smoothness',
+    'Aesthetic Quality',
+    'Image Quality',
+    'Overall Consistency',
+    'I2V Subject',
+    'I2V Background'
+]
+
+# Columns that can never be deselected
+PREDICT_NEVER_HIDDEN_COLUMNS = ['model', 'Overall']
+
+# Columns displayed by default (using renamed column names)
+PREDICT_DEFAULT_DISPLAYED_COLUMNS = PREDICT_NEVER_HIDDEN_COLUMNS + PREDICT_ALL_SELECTED_COLUMNS
+
+def load_predict_csv(csv_path):
+    """Load CSV and apply column ordering"""
+    df = pd.read_csv(csv_path)
+
+    # Remove specified columns
+    df = df.drop(columns=PREDICT_COLUMNS_TO_REMOVE, errors='ignore')
+
+    # Rename columns according to mapping
+    df = df.rename(columns=PREDICT_COLUMN_NAME_MAPPING)
+
+    # Reorder columns (only keep columns that exist in the dataframe)
+    available_cols = [col for col in PREDICT_COLUMN_ORDER if col in df.columns]
+    df = df[available_cols]
+
+    # Convert model names to HuggingFace links
+    if 'model' in df.columns:
+        df['model'] = df['model'].apply(create_model_link)
+
+    # Format numbers to ensure decimal places (1 decimal for numeric columns)
+    for col in df.columns:
+        if col not in ['model', 'params', 'activate_params'] and pd.api.types.is_numeric_dtype(df[col]):
+            df[col] = df[col].apply(lambda x: f"{x:.1f}" if pd.notna(x) else x)
+
+    return df
+
+
+def select_predict_domain_score():
+    """Return domain score for checkbox selection"""
+    return gr.update(value=PREDICT_DOMAIN_SCORE_DIMENSIONS)
+
+def select_predict_quality_score():
+    """Return quality score for checkbox selection"""
+    return gr.update(value=PREDICT_QUALITY_SCORE_DIMENSIONS)
+
+def deselect_predict_all():
+    """Deselect all dimensions"""
+    return gr.update(value=PREDICT_DESELECTED_COLUMNS)
+
+def select_predict_all():
+    """Select all dimensions"""
+    return gr.update(value=PREDICT_ALL_SELECTED_COLUMNS)
+
+def on_predict_dimension_selection_change(selected_columns, full_df):
+    """Handle dimension selection changes and update the dataframe"""
+    # Always include model and Overall columns
+    present_columns = ['model', 'Overall']
+
+    # Add selected columns
+    for col in selected_columns:
+        if col not in present_columns and col in full_df.columns:
+            present_columns.append(col)
+
+    # Filter dataframe to show only selected columns
+    updated_data = full_df[present_columns]
+
+    # Determine datatypes
+    datatypes = []
+    for col in present_columns:
+        if col == 'model':
+            datatypes.append('markdown')
+        elif col in ['params', 'activate_params']:
+            datatypes.append('number')
+        else:
+            datatypes.append('str')
+
+    return gr.update(value=updated_data, datatype=datatypes, headers=present_columns)
+
+
+def init_predict_leaderboard(dataframe):
+    """Initialize the Predict leaderboard with given dataframe"""
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    # Get columns that exist in the dataframe
+    available_default_cols = [col for col in PREDICT_DEFAULT_DISPLAYED_COLUMNS if col in dataframe.columns]
+
+    # Filter dataframe to show only default columns initially
+    display_df = dataframe[available_default_cols]
+
+    # Determine datatypes dynamically
+    datatypes = []
+    for col in display_df.columns:
+        if col == 'model':
+            datatypes.append('markdown')
+        elif col in ['params', 'activate_params']:
+            datatypes.append('number')
+        else:
+            datatypes.append('str')  # All numeric columns are now formatted as strings
+
+    # Create the UI components
+    with gr.Row():
+        with gr.Column(scale=1):
+            domain_score_btn = gr.Button("Domain Score", size="md")
+            quality_score_btn = gr.Button("Quality Score", size="md")
+            select_all_btn = gr.Button("Select All", size="md")
+            deselect_btn = gr.Button("Deselect All", size="md")
+
+        with gr.Column(scale=4):
+            # Get all dimension columns (exclude model, Overall, scores, and params)
+            dimension_choices = [col for col in dataframe.columns
+                                if col not in PREDICT_NEVER_HIDDEN_COLUMNS + PREDICT_HIDDEN_COLUMNS]
+
+            checkbox_group = gr.CheckboxGroup(
+                choices=dimension_choices,
+                value=[col for col in PREDICT_DEFAULT_DISPLAYED_COLUMNS if col in dimension_choices],
+                label="Evaluation Dimensions",
+                interactive=True,
+            )
+
+    data_component = gr.Dataframe(
+        value=display_df,
+        headers=list(display_df.columns),
+        datatype=datatypes,
+        interactive=False,
+        visible=True,
+        wrap=False,
+        column_widths=["320px"] + ["200px"] * (len(display_df.columns) - 1),
+        pinned_columns=1,
+    )
+
+    # Setup event handlers
+    domain_score_btn.click(
+        select_predict_domain_score,
+        inputs=None,
+        outputs=[checkbox_group]
+    ).then(
+        fn=on_predict_dimension_selection_change,
+        inputs=[checkbox_group, gr.State(dataframe)],
+        outputs=data_component
+    )
+
+    quality_score_btn.click(
+        select_predict_quality_score,
+        inputs=None,
+        outputs=[checkbox_group]
+    ).then(
+        fn=on_predict_dimension_selection_change,
+        inputs=[checkbox_group, gr.State(dataframe)],
+        outputs=data_component
+    )
+
+    deselect_btn.click(
+        deselect_predict_all,
+        inputs=None,
+        outputs=[checkbox_group]
+    ).then(
+        fn=on_predict_dimension_selection_change,
+        inputs=[checkbox_group, gr.State(dataframe)],
+        outputs=data_component
+    )
+
+    select_all_btn.click(
+        select_predict_all,
+        inputs=None,
+        outputs=[checkbox_group]
+    ).then(
+        fn=on_predict_dimension_selection_change,
+        inputs=[checkbox_group, gr.State(dataframe)],
+        outputs=data_component
+    )
+
+    checkbox_group.change(
+        fn=on_predict_dimension_selection_change,
+        inputs=[checkbox_group, gr.State(dataframe)],
+        outputs=data_component
+    )
+
+    return data_component
+
+
+# ============================================================================
+# Reason Tab Configuration and Utilities
+# ============================================================================
+
+# Column name mapping for display
+REASON_COLUMN_MAPPING = {
+    'Physical world': 'Physics'
+}
+
+# Desired column order
+REASON_COLUMN_ORDER = [
+    'model',
+    'Overall',
+    'Common Sense',
+    'Embodied Reasoning',
+    'Space',
+    'Time',
+    'Physics',
+    'BridgeData V2',
+    'RoboVQA',
+    'RoboFail',
+    'Agibot',
+    'HoloAssist',
+    'AV',
+    'params',
+    'activate_params'
+]
+
+# Columns to hide by default (but still available for filtering/selection)
+REASON_HIDDEN_COLUMNS = ['params', 'activate_params']
+
+# Reasoning dimensions (for selection button)
+REASON_COMMON_SENSE_DIMENSIONS = [
+    'Common Sense',
+    'Space',
+    'Time',
+    'Physics',
+]
+
+# Domain dimensions (for selection button)
+REASON_EMBODIED_REASONING_DIMENSIONS = [
+    'Embodied Reasoning',
+    'Space',
+    'Time',
+    'Physics',
+    'BridgeData V2',
+    'RoboVQA',
+    'RoboFail',
+    'Agibot',
+    'HoloAssist',
+    'AV',
+]
+
+REASON_DESELECTED_COLUMNS = [
+    'Common Sense',
+    'Embodied Reasoning',
+]
+
+REASON_ALL_SELECTED_COLUMNS = [
+    'Common Sense',
+    'Embodied Reasoning',
+    'Space',
+    'Time',
+    'Physics',
+    'BridgeData V2',
+    'RoboVQA',
+    'RoboFail',
+    'Agibot',
+    'HoloAssist',
+    'AV',
+]
+
+# Columns that can never be deselected
+REASON_NEVER_HIDDEN_COLUMNS = ['model', 'Overall']
+
+# Columns displayed by default (using renamed column names)
+REASON_DEFAULT_DISPLAYED_COLUMNS = REASON_NEVER_HIDDEN_COLUMNS + REASON_ALL_SELECTED_COLUMNS
+
+
+def load_reason_csv(csv_path):
+    """Load CSV and apply column mapping and ordering"""
+    df = pd.read_csv(csv_path)
+
+    # Apply column mapping
+    df = df.rename(columns=REASON_COLUMN_MAPPING)
+
+    # Reorder columns (only keep columns that exist in the dataframe)
+    available_cols = [col for col in REASON_COLUMN_ORDER if col in df.columns]
+    df = df[available_cols]
+
+    # Convert model names to HuggingFace links
+    if 'model' in df.columns:
+        df['model'] = df['model'].apply(create_model_link)
+
+    # Format numbers to ensure decimal places (1 decimal for integers)
+    for col in df.columns:
+        if col not in ['model', 'params', 'activate_params'] and pd.api.types.is_numeric_dtype(df[col]):
+            df[col] = df[col].apply(lambda x: f"{x:.1f}" if pd.notna(x) else x)
+
+    return df
+
+
+def select_reason_common_sense_dimensions():
+    """Return reasoning dimensions for checkbox selection"""
+    return gr.update(value=REASON_COMMON_SENSE_DIMENSIONS)
+
+
+def select_reason_embodied_reasoning_dimensions():
+    """Return domain dimensions for checkbox selection"""
+    return gr.update(value=REASON_EMBODIED_REASONING_DIMENSIONS)
+
+
+def deselect_reason_all():
+    """Deselect all dimensions"""
+    return gr.update(value=REASON_DESELECTED_COLUMNS)
+
+
+def select_reason_all():
+    """Select all dimensions"""
+    return gr.update(value=REASON_ALL_SELECTED_COLUMNS)
+
+
+def on_reason_dimension_selection_change(selected_columns, full_df):
+    """Handle dimension selection changes and update the dataframe"""
+    # Always include model and Overall columns
+    present_columns = ['model', 'Overall']
+
+    # Add selected columns
+    for col in selected_columns:
+        if col not in present_columns and col in full_df.columns:
+            present_columns.append(col)
+
+    # Filter dataframe to show only selected columns
+    updated_data = full_df[present_columns]
+
+    # Determine datatypes
+    datatypes = []
+    for col in present_columns:
+        if col == 'model':
+            datatypes.append('markdown')
+        elif col in ['params', 'activate_params']:
+            datatypes.append('number')
+        else:
+            datatypes.append('str')
+
+    return gr.update(value=updated_data, datatype=datatypes, headers=present_columns)
+
+
+def init_reason_leaderboard(dataframe):
+    """Initialize the Reason leaderboard with given dataframe"""
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    # Get columns that exist in the dataframe
+    available_default_cols = [col for col in REASON_DEFAULT_DISPLAYED_COLUMNS if col in dataframe.columns]
+
+    # Filter dataframe to show only default columns initially
+    display_df = dataframe[available_default_cols]
+
+    # Determine datatypes dynamically
+    datatypes = []
+    for col in display_df.columns:
+        if col == 'model':
+            datatypes.append('markdown')
+        elif col in ['params', 'activate_params']:
+            datatypes.append('number')
+        else:
+            datatypes.append('str')  # All numeric columns are now formatted as strings
+
+    # Create the UI components
+    with gr.Row():
+        with gr.Column(scale=1):
+            common_sense_btn = gr.Button("Common Sense", size="md")
+            embodied_reasoning_btn = gr.Button("Embodied Reasoning", size="md")
+            select_all_btn = gr.Button("Select All", size="md")
+            deselect_btn = gr.Button("Deselect All", size="md")
+
+        with gr.Column(scale=4):
+            # Get all dimension columns (exclude model, Overall, and params)
+            dimension_choices = [col for col in dataframe.columns
+                                if col not in REASON_NEVER_HIDDEN_COLUMNS + REASON_HIDDEN_COLUMNS]
+
+            checkbox_group = gr.CheckboxGroup(
+                choices=dimension_choices,
+                value=[col for col in REASON_DEFAULT_DISPLAYED_COLUMNS if col in dimension_choices],
+                label="Evaluation Dimensions",
+                interactive=True,
+            )
+
+    data_component = gr.Dataframe(
+        value=display_df,
+        headers=list(display_df.columns),
+        datatype=datatypes,
+        interactive=False,
+        visible=True,
+        wrap=False,  # Allow horizontal scrolling, don't wrap content
+        column_widths=["320px"] + ["200px"] * (len(display_df.columns) - 1),
+        pinned_columns=1,
+    )
+
+    # Setup event handlers
+    common_sense_btn.click(
+        select_reason_common_sense_dimensions,
+        inputs=None,
+        outputs=[checkbox_group]
+    ).then(
+        fn=on_reason_dimension_selection_change,
+        inputs=[checkbox_group, gr.State(dataframe)],
+        outputs=data_component
+    )
+
+    embodied_reasoning_btn.click(
+        select_reason_embodied_reasoning_dimensions,
+        inputs=None,
+        outputs=[checkbox_group]
+    ).then(
+        fn=on_reason_dimension_selection_change,
+        inputs=[checkbox_group, gr.State(dataframe)],
+        outputs=data_component
+    )
+
+    deselect_btn.click(
+        deselect_reason_all,
+        inputs=None,
+        outputs=[checkbox_group]
+    ).then(
+        fn=on_reason_dimension_selection_change,
+        inputs=[checkbox_group, gr.State(dataframe)],
+        outputs=data_component
+    )
+
+    select_all_btn.click(
+        select_reason_all,
+        inputs=None,
+        outputs=[checkbox_group]
+    ).then(
+        fn=on_reason_dimension_selection_change,
+        inputs=[checkbox_group, gr.State(dataframe)],
+        outputs=data_component
+    )
+
+    checkbox_group.change(
+        fn=on_reason_dimension_selection_change,
+        inputs=[checkbox_group, gr.State(dataframe)],
+        outputs=data_component
+    )
+
+    return data_component
+
+
+# ============================================================================
+# Main Application
+# ============================================================================
+
+demo = gr.Blocks()
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🎨 Predict", elem_id="predict-tab", id=0):
+            # Load data for Predict tab
+            predict_df = load_predict_csv("data/predict-leaderboard.csv")
+            predict_leaderboard = init_predict_leaderboard(predict_df)
+
+        with gr.TabItem("🔄 Transfer", elem_id="transfer-tab", id=1):
+            gr.Markdown("## Coming Soon", elem_classes="markdown-text")
+
+        with gr.TabItem("🧠 Reason", elem_id="reason-tab", id=2):
+            # Load data for Reason tab
+            reason_df = load_reason_csv("data/reason-leaderboard.csv")
+            reason_leaderboard = init_reason_leaderboard(reason_df)
+
+        with gr.TabItem("ℹ️ About", elem_id="about-tab", id=3):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+demo.launch()

data/predict-leaderboard.csv

@@ -0,0 +1,5 @@
+model,params,activate_params,Overall,AV,Common,Human,Industry,Misc,Physics,Robot,Avg Score/Video,Common+Misc,Domain Score,Aesthetic,BG Consistency,Image Quality,Motion,Overall Consistency,Subject Consistency,I2V BG,I2V Subject,Quality Score
+nvidia/Cosmos-Predict2.5-2B,2.0,2.0,81.0,66.1,95.9,81.4,87.8,91.0,93.9,80.8,84.4,94.1,84.0,52.4,94.2,70.8,99.1,20.1,92.5,97.4,96.6,77.9
+Wan-AI/Wan2.2-I2V-A14B,14.0,14.0,80.6,66.3,94.6,82.1,89.2,90.9,91.8,81.7,84.5,93.2,84.1,51.2,93.7,69.6,98.3,20.4,91.6,96.6,96.0,77.2
+Wan-AI/Wan2.2-TI2V-5B,5.0,5.0,80.4,65.2,95.3,83.0,88.4,89.6,91.5,79.3,84.1,93.1,83.4,51.9,93.7,69.9,98.8,20.3,91.8,96.7,95.9,77.4
+Wan-AI/Wan2.1-I2V-14B-720P,14.0,14.0,79.7,66.9,93.7,80.1,89.7,85.5,88.7,80.1,82.9,90.6,82.7,51.5,93.1,70.1,98.1,20.4,90.0,96.0,95.2,76.8

data/reason-leaderboard.csv

@@ -0,0 +1,10 @@
+model,params,activate_params,Overall,AV,Agibot,BridgeData V2,Common Sense,Embodied Reasoning,HoloAssist,Physics,RoboFail,RoboVQA,Space,Time
+Qwen/Qwen3-VL-30B-A3B-Instruct,30.0,3.0,60.6,49.0,43.0,36.0,59.9,61.3,81.0,59.7,67.0,89.1,52.5,62.1
+Qwen/Qwen2.5-VL-72B-Instruct,72.0,72.0,56.8,39.0,35.0,35.0,57.9,55.7,58.0,52.2,73.0,90.9,56.2,62.8
+nvidia/Cosmos-Reason1-7B,7.0,7.0,54.3,47.0,42.0,41.0,50.7,57.9,57.0,44.2,65.0,91.8,57.5,53.7
+Qwen/Qwen2.5-VL-32B-Instruct,32.0,32.0,51.9,33.0,34.0,32.0,53.8,50.0,55.0,45.6,52.0,90.0,50.0,61.1
+Qwen/Qwen2.5-VL-7B-Instruct,7.0,7.0,50.3,45.0,44.0,33.0,47.7,53.0,47.0,37.6,62.0,83.6,47.5,55.4
+Qwen/Qwen2.5-VL-3B-Instruct,3.0,3.0,48.1,29.0,36.0,31.0,47.4,48.9,48.0,42.9,63.0,82.7,47.5,50.7
+Qwen/Qwen2-VL-2B-Instruct,2.0,2.0,40.0,51.0,24.0,25.0,44.5,35.4,28.0,41.2,34.0,49.1,32.5,50.3
+Qwen/Qwen2-VL-72B-Instruct,72.0,72.0,40.0,25.0,31.0,28.0,45.0,34.9,21.0,40.3,49.0,53.6,50.0,47.3
+Qwen/Qwen2-VL-7B-Instruct,7.0,7.0,38.8,24.0,28.0,28.0,44.5,33.1,26.0,44.7,38.0,52.7,38.8,46.0

requirements.txt

@@ -0,0 +1,2 @@
+gradio
+pandas