Initial OpenHands Index leaderboard based on ASTA Bench

- Updated branding (title, README, logo)
- Changed dataset references to OpenHands Index datasets
- Modified agenteval.json to include 6 OpenHands datasets:
* swe-bench
* multi-swe-bench
* swe-bench-multimodal
* swt-bench
* commit0
* gaia
- Added mock data for initial testing

Co-authored-by: openhands <openhands@all-hands.dev>

.gitattributes

@@ -0,0 +1 @@
+

.github/workflows/integration-tests.yml

@@ -0,0 +1,44 @@
+name: Integration Tests
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  integration-test:
+    runs-on: ubuntu-latest
+    
+    environment:
+      name: testing
+      
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        lfs: true
+    
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+    
+    - name: Cache pip dependencies
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r requirements-dev.txt
+
+    - name: Run integration tests
+      run: |
+        pytest tests/integration/ -v --tb=short
+      env:
+        HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        HF_CONFIG: continuous-integration
+        IS_INTERNAL: true

.gitignore

@@ -0,0 +1,184 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# PyPI configuration file
+.pypirc
+
+# Vim files
+*.swp
+*.swo
+*.un~
+
+# Misc
+.DS_Store
+.mise.toml
+.vscode/
+.gradio/
+
+.claude

Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.10-slim
+
+
+# (0) Install SSH client tools (and git, if you're pulling via SSH)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends openssh-client git && \
+    rm -rf /var/lib/apt/lists/*
+
+# The two following lines are requirements for the Dev Mode to be functional
+# Learn more about the Dev Mode at https://huggingface.co/dev-mode-explorers
+RUN useradd -m -u 1000 user
+WORKDIR /app
+
+
+# (2) Copy dependencies manifest
+COPY --chown=user requirements.txt requirements.txt
+
+# (3) Install dependencies, mounting SSH keys and optional HTTPS creds
+RUN --mount=type=secret,id=AGENTEVAL_DEPLOY_KEY,mode=0400,required=true \
+    --mount=type=secret,id=ASTABENCH_DEPLOY_KEY,mode=0400,required=true \
+    mkdir -p /root/.ssh && chmod 700 /root/.ssh && \
+    cat /run/secrets/AGENTEVAL_DEPLOY_KEY > /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519 && \
+    cat /run/secrets/ASTABENCH_DEPLOY_KEY > /root/.ssh/id_astabench && chmod 600 /root/.ssh/id_astabench && \
+    ssh-keyscan github.com >> /root/.ssh/known_hosts && \
+    printf 'Host github.com\n  User git\n  IdentityFile /root/.ssh/id_ed25519\n  IdentityFile /root/.ssh/id_astabench\n  StrictHostKeyChecking no\n' >> /root/.ssh/config && \
+    # rewrite all GitHub HTTPS URLs to SSH so nested deps install via SSH
+    git config --global url."ssh://git@github.com/".insteadOf "https://github.com/" && \
+    pip install --no-cache-dir --upgrade -r requirements.txt
+
+# (4) Copy in your Gradio app code
+COPY . .
+RUN mkdir -p /home/user/data && chown -R user:user /home/user/data
+
+# Make the app treat this as non‑debug (so DATA_DIR=/home/user/data)
+ENV system=spaces
+
+# (5) Switch to a non-root user
+USER user
+
+# (6) Expose Gradio’s default port
+EXPOSE 7860
+
+# (7) Launch your app
+CMD ["python", "app.py"]

README.md

@@ -0,0 +1,60 @@
+---
+title: OpenHands Index
+emoji: 🤖
+colorFrom: blue
+colorTo: purple
+sdk: docker
+app_file: app.py
+pinned: true
+license: apache-2.0
+hf_oauth: true
+app_port: 7860
+failure_strategy: none
+tags:
+  - leaderboard
+---
+
+## OpenHands Index Leaderboard
+
+This leaderboard tracks agent performance across multiple software engineering and AI benchmarks.
+
+## Development
+The leaderboard is built using the [HuggingFace Datasets](https://huggingface.co/docs/datasets/index) library, which provides a convenient way to manage and query datasets.
+Results are sourced from the [OpenHands Index Results](https://github.com/OpenHands/openhands-index-results) repository.
+
+To run the leaderboard locally first make sure to set this env variable:
+```bash
+export IS_INTERNAL=true
+```
+You can then start it up with the following command:
+```bash
+python app.py
+```
+This will start a local server that you can access in your web browser at `http://localhost:7860`.
+
+## Hugging Face Integration
+The repo backs two Hugging Face leaderboard spaces:
+- https://huggingface.co/spaces/allenai/asta-bench-internal-leaderboard
+- https://huggingface.co/spaces/allenai/asta-bench-leaderboard
+
+Please follow the steps below to push changes to the leaderboards on Hugging Face.
+
+Before pushing, make sure to merge your changes to the `main` branch of this repository. (following the standard GitHub workflow of creating a branch, making changes, and then merging it back to `main`).
+
+Before pushing for the first time, you'll need to add the Hugging Face remote repositories if you haven't done so already. You can do this by running the following commands:
+
+```bash
+git remote add huggingface https://huggingface.co/spaces/allenai/asta-bench-internal-leaderboard
+git remote add huggingface-public https://huggingface.co/spaces/allenai/asta-bench-leaderboard
+```
+You can verify that the remotes have been added by running:
+
+```bash
+git remote -v
+```
+Then, to push the changes to the Hugging Face leaderboards, you can use the following commands:
+
+```bash
+git push huggingface main:main   
+git push huggingface-public main:main
+```

about.py

@@ -0,0 +1,144 @@
+import gradio as gr
+
+
+def build_page():
+    with gr.Column(elem_id="about-page-content-wrapper"):
+        # --- Section 1: About AstaBench ---
+        gr.HTML(
+            """
+            <h2>About AstaBench</h2>
+            <p>
+                AstaBench is a novel AI agents evaluation framework, providing a challenging new test for AI agents: the first benchmark challenge that evaluates agents’ scientific abilities on a broad spectrum of research skills, including literature understanding, data analysis, planning, tool use, coding, and search. Asta’s set of standard tools makes it easy to build general-purpose science agents and to compare their performance in an apples-to-apples manner.
+            </p>
+            """
+        )
+        gr.Markdown("---", elem_classes="divider-line")
+
+        # --- Section 2: Why AstaBench? ---
+        gr.HTML(
+            """
+            <h2>Why AstaBench?</h2>
+            <p>
+                Most current benchmarks test agentic AI and isolated aspects of scientific reasoning, but rarely evaluate AI agentic behavior rigorously or capture the full skill set scientific research requires. Agents can appear effective despite inconsistent results and high compute use, often outperforming others by consuming more resources. Advancing scientific AI requires evaluations that emphasize reproducibility, efficiency, and the real complexity of research.
+            </p>
+            <br>
+            <p>
+                AstaBench fills this gap: an agents evaluation framework and suite of open benchmarks for evaluating scientific AI assistants on core scientific tasks that require novel reasoning. AstaBench helps scientists identify which agents best support their needs through task-relevant leaderboards, while giving AI developers a standard execution environment and tools to test the scientific reasoning capabilities of their agents compared to well-known baselines from the literature, including both open and closed LLM foundation models.
+            </p>
+            """
+        )
+        gr.Markdown("---", elem_classes="divider-line")
+
+        # --- Section 3: What Does AstaBench Include? ---
+        gr.HTML(
+            """
+            <h2>What Does AstaBench Include?</h2>
+            <p>
+                AstaBench includes a rigorous agents evaluation framework and a suite of benchmarks consisting of over 2,400 problems across 11 benchmarks, organized into four core categories:
+            </p>
+            <ul class="info-list">
+                <li>Literature Understanding</li>
+                <li>Code & Execution</li>
+                <li>Data Analysis</li>
+                <li>End-to-End Discovery</li>
+            </ul>
+            <p>
+                Plus: a large suite of integrated agents and leaderboards with results from extensive evaluation of agents and models.
+            </p>
+            <p>
+                🔍 Learn more in the <a href="https://allenai.org/blog/astabench" target="_blank" class="primary-link-button">AstaBench technical blog post</a>
+            </p>
+            """
+        )
+        gr.Markdown("---", elem_classes="divider-line")
+
+        # --- Section 4: Understanding the Leaderboards ---
+        gr.HTML(
+            """
+            <h2>Understanding the Leaderboards</h2>
+            <p>
+                The AstaBench Overall Leaderboard provides a high-level view of overall agent performance and efficiency:
+            </p>
+            <ul class="info-list">
+                <li>Overall score: A macro-average of the four category-level averages (equal weighting)</li>
+                <li>Overall cost: Average cost per task, aggregated only across benchmarks with reported cost</li>
+            </ul>
+            <p>
+                Each category leaderboard provides:
+            </p>
+            <ul class="info-list">
+                <li>Average score and cost for that category (macro-averaged across the benchmarks in the category)</li>
+                <li>A breakdown by individual benchmarks</li>
+            </ul>
+            """
+        )
+        gr.Markdown("---", elem_classes="divider-line")
+
+        # --- Section 5: Scoring & Aggregation ---
+        gr.HTML(
+            """
+            <h2>Scoring & Aggregation</h2>
+            <p>
+                AstaBench encourages careful, transparent evaluation. Here's how we handle scoring, cost, and partial results:
+            </p>
+            
+            <h3>Scores</h3>
+            <ul class="info-list">
+                <li>Each benchmark returns an average score based on per-problem scores</li>
+                <li>All scores are aggregated upward using macro-averaging</li>
+                <li>Partial completions are included (even with poor performance)</li>
+            </ul>
+            
+            <h3>Cost</h3>
+            <ul class="info-list">
+                <li>Costs are reported in USD per task.</li>
+                <li>Benchmarks without cost data are excluded from cost averages</li>
+                <li>In scatter plots, agents without cost are plotted to the far right and clearly marked.</li>
+            </ul>
+            
+            <p>
+                <em>Note: Cost values reflect pricing and infrastructure conditions at a fixed point in time. We recognize that compute costs may change over time and vary by provider, and are actively working on methods to keep costs up-to-date and normalized for fair comparisons.</em>
+            </p>
+            
+            <h3>Coverage</h3>
+            <ul class="info-list">
+                <li>Main leaderboard: category coverage (X/4)</li>
+                <li>Category view: benchmark coverage (X/Y)</li>
+                <li>Incomplete coverage is flagged visually</li>
+            </ul>
+            
+            <p>
+                These design choices ensure fair comparison while penalizing cherry-picking and omissions.
+            </p>
+            """
+        )
+        gr.Markdown("---", elem_classes="divider-line")
+
+        # --- Section 6: Learn More ---
+        gr.HTML(
+            """
+            <div class="learn-more-section">
+                <h2>Learn More</h2>
+                <div class="link-buttons-container">
+                    
+                    <a href="https://allenai.org/blog/astabench" target="_blank" class="link-button">
+                        <span style="color:#0fcb8c;">AstaBench technical blog post</span>
+                        <span class="external-link-icon">↗</span>
+                    </a>
+                    
+                    <a href="/submit" target="_blank" class="link-button">
+                        <span style="color:#0fcb8c;">Submit an agent for evaluation</span>
+                        <span class="external-link-icon">↗</span>
+                    </a>
+                    
+                </div>
+            </div>
+            """
+        )
+        # Floating feedback button
+        floating_feedback_button_html = """
+        <div>
+            <a id="feedback-button" href="https://docs.google.com/forms/d/e/1FAIpQLSfJdVkD62aPYh8XehN2FrSeHUWt488Ejc-QdtuZn5NZ3eNoxA/viewform">Have feedback?</a>
+        </div>
+        """
+        gr.HTML(floating_feedback_button_html)

aliases.py

@@ -0,0 +1,23 @@
+from agenteval.config import (
+    OPENNESS_OPEN_SOURCE_OPEN_WEIGHTS as CANONICAL_OPENNESS_OPEN_SOURCE_OPEN_WEIGHTS,
+    OPENNESS_OPEN_SOURCE_CLOSED_WEIGHTS as CANONICAL_OPENNESS_OPEN_SOURCE_CLOSED_WEIGHTS,
+    OPENNESS_CLOSED_API_AVAILABLE as CANONICAL_OPENNESS_CLOSED_API_AVAILABLE,
+    OPENNESS_CLOSED_UI_ONLY as CANONICAL_OPENNESS_CLOSED_UI_ONLY,
+    TOOL_USAGE_STANDARD as CANONICAL_TOOL_USAGE_STANDARD,
+    TOOL_USAGE_CUSTOM_INTERFACE as CANONICAL_TOOL_USAGE_CUSTOM_INTERFACE,
+    TOOL_USAGE_FULLY_CUSTOM as CANONICAL_TOOL_USAGE_FULLY_CUSTOM,
+)
+
+
+OPENNESS_ALIASES = {
+    CANONICAL_OPENNESS_OPEN_SOURCE_OPEN_WEIGHTS: {"Open Source + Open Weights"},
+    CANONICAL_OPENNESS_OPEN_SOURCE_CLOSED_WEIGHTS: {"Open Source"},
+    CANONICAL_OPENNESS_CLOSED_API_AVAILABLE: {"API Available"},
+    CANONICAL_OPENNESS_CLOSED_UI_ONLY: {"Closed"}
+}
+
+TOOL_USAGE_ALIASES = {
+    CANONICAL_TOOL_USAGE_STANDARD: {},
+    CANONICAL_TOOL_USAGE_CUSTOM_INTERFACE: {"Custom with Standard Search"},
+    CANONICAL_TOOL_USAGE_FULLY_CUSTOM: {"Fully Custom"}
+}

app.py

@@ -0,0 +1,282 @@
+# app.py
+import logging
+
+logging.basicConfig(level=logging.WARNING)
+
+import gradio as gr
+import urllib.parse
+
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import HfApi
+
+from config import LEADERBOARD_PATH, LOCAL_DEBUG
+from content import css
+from main_page import build_page as build_main_page
+from literature_understanding import build_page as build_lit_page
+from c_and_e import build_page as build_c_and_e_page
+from data_analysis import build_page as build_data_analysis_page
+from e2e import build_page as build_e2e_page
+from submission import build_page as build_submission_page
+from about import build_page as build_about_page
+
+api = HfApi()
+LOGO_PATH = "assets/logo.svg"
+# JavaScripts
+scroll_script = """
+<script>
+function scroll_to_element(id) {
+    console.log("Global scroll_to_element called for ID:", id);
+    const element = document.querySelector('#' + id);
+    if (element) {
+        console.log("Element found:", element);
+        element.scrollIntoView({ behavior: 'smooth', block: 'start' });
+    } else {
+        console.error("Error: Element with ID '" + id + "' not found in the document.");
+    }
+}
+</script>
+"""
+redirect_script = """
+<script>
+    if (window.location.pathname === '/') { window.location.replace('/home'); }
+</script>
+"""
+tooltip_script = """
+<script>
+function initializeSmartTooltips() {
+    // Find all tooltip trigger icons
+    const tooltipIcons = document.querySelectorAll('.tooltip-icon-legend');
+
+    tooltipIcons.forEach(icon => {
+        // Find the tooltip card associated with this icon
+        const tooltipCard = icon.querySelector('.tooltip-card');
+        if (!tooltipCard) return;
+
+        // Move the card to the end of the <body>. This is the KEY to escaping
+        // any parent containers that might clip it.
+        document.body.appendChild(tooltipCard);
+
+        // --- MOUSE HOVER EVENT ---
+        icon.addEventListener('mouseenter', () => {
+            // Get the exact position of the icon on the screen
+            const iconRect = icon.getBoundingClientRect();
+            // Get the dimensions of the tooltip card
+            const cardRect = tooltipCard.getBoundingClientRect();
+
+            // Calculate the ideal top position (above the icon with a 10px gap)
+            const top = iconRect.top - cardRect.height - 10;
+            
+            // --- Smart Centering Logic ---
+            // Start by calculating the perfect center
+            let left = iconRect.left + (iconRect.width / 2) - (cardRect.width / 2);
+
+            // Check if it's going off the left edge of the screen
+            if (left < 10) {
+                left = 10; // Pin it to the left with a 10px margin
+            }
+            // Check if it's going off the right edge of the screen
+            if (left + cardRect.width > window.innerWidth) {
+                left = window.innerWidth - cardRect.width - 10; // Pin it to the right
+            }
+
+            // Apply the calculated position and show the card
+            tooltipCard.style.top = `${top}px`;
+            tooltipCard.style.left = `${left}px`;
+            tooltipCard.classList.add('visible');
+        });
+
+        // --- MOUSE LEAVE EVENT ---
+        icon.addEventListener('mouseleave', () => {
+            // Hide the card
+            tooltipCard.classList.remove('visible');
+        });
+    });
+}
+
+// Poll the page until the tooltips exist, then run the initialization.
+const tooltipInterval = setInterval(() => {
+    if (document.querySelector('.tooltip-icon-legend')) {
+        clearInterval(tooltipInterval);
+        initializeSmartTooltips();
+    }
+}, 200);
+</script>
+"""
+redirect_submission_on_close_script = """
+<script>
+function initializeRedirectObserver() {
+    const successModal = document.querySelector('#success-modal');
+    
+    if (successModal) {
+        const observer = new MutationObserver((mutationsList) => {
+            for (const mutation of mutationsList) {
+                // We only care about changes to the 'class' attribute.
+                if (mutation.type === 'attributes' && mutation.attributeName === 'class') {
+                    
+                    // Check if the 'hide' class has been ADDED to the class list.
+                    // This is how Gradio hides the modal.
+                    if (successModal.classList.contains('hide')) {
+                        console.log("Success modal was closed. Redirecting to homepage...");
+                        // This is the command to redirect the browser.
+                        window.location.href = '/home';
+                    }
+                }
+            }
+        });
+
+        // Tell the observer to watch the modal for attribute changes.
+        observer.observe(successModal, { attributes: true });
+    }
+}
+
+// Polling mechanism to wait for Gradio to build the UI.
+const redirectInterval = setInterval(() => {
+    if (document.querySelector('#success-modal')) {
+        clearInterval(redirectInterval);
+        initializeRedirectObserver();
+    }
+}, 200);
+</script>
+"""
+# --- Theme Definition ---
+theme = gr.themes.Base(
+    primary_hue=gr.themes.Color(c100="#CFF5E8", c200="#B7EFDD", c300="#9FEAD1", c400="#87E5C5", c50="#E7FAF3", c500="#6FE0BA", c600="#57DBAF", c700="#3FD5A3", c800="#27D09C", c900="#0FCB8C", c950="#0fcb8c"),
+    secondary_hue=gr.themes.Color(c100="#FCDCEB", c200="#FBCBE1", c300="#F9BAD7", c400="#F7A8CD", c50="#FDEEF5", c500="#F697C4", c600="#F586BA", c700="#F375B0", c800="#F263A6", c900="#F0529C", c950="#F0529C"),
+    neutral_hue=gr.themes.Color(c100="#FDF9F4", c200="#C9C9C3", c300="#B0B5AF", c400="#97A09C", c50="#FAF2E9", c500="#7F8C89", c600="#667876", c700="#344F4F", c800="#1C3A3C", c900="#032629", c950="032629"),
+    font=[gr.themes.GoogleFont('Manrope'), 'ui-sans-serif', 'sans-serif', 'sans-serif'],
+    font_mono=[gr.themes.GoogleFont('Roboto Mono'), 'ui-monospace', 'monospace', 'monospace'],
+).set(
+    body_text_color='*neutral_950',
+    body_text_color_subdued='*neutral_950',
+    body_text_color_subdued_dark='*neutral_50',
+    body_text_color_dark='*neutral_50',
+    background_fill_primary='*neutral_50',
+    background_fill_primary_dark='*neutral_900',
+    background_fill_secondary='*neutral_100',
+    background_fill_secondary_dark='*neutral_800',
+    border_color_accent='*secondary_900',
+    border_color_accent_subdued='*neutral_400',
+    border_color_accent_subdued_dark='*neutral_400',
+    color_accent='*primary_900',
+    color_accent_soft='*neutral_200',
+    color_accent_soft_dark='*neutral_800',
+    link_text_color='*secondary_900',
+    link_text_color_dark='*primary_900',
+    link_text_color_active_dark='*primary_600',
+    link_text_color_hover_dark='*primary_700',
+    link_text_color_visited_dark='*primary_600',
+    table_even_background_fill='*neutral_100',
+    table_even_background_fill_dark='*neutral_800',
+    button_primary_background_fill='*secondary_900',
+    button_primary_background_fill_dark='*primary_900',
+    button_primary_background_fill_hover='*secondary_600',
+    button_primary_background_fill_hover_dark='*primary_600',
+    button_secondary_background_fill="#9FEAD1",
+    button_secondary_background_fill_dark="#9FEAD1",
+    button_secondary_text_color="*neutral_900",
+    button_secondary_text_color_dark="*neutral_900",
+    block_title_text_color="*neutral_900",
+    button_primary_text_color='*neutral_900',
+    block_title_text_color_dark="#ffffff",
+    button_primary_text_color_dark='*neutral_900',
+    block_border_color="#032629",
+    block_border_color_dark="#9fead1",
+    block_background_fill_dark="#032629",
+    block_background_fill="#FAF2E9",
+    checkbox_label_text_color="#032629",
+    checkbox_label_background_fill="#D8D6CF",
+    checkbox_label_background_fill_dark="#254243",
+    checkbox_background_color_selected="#F0529C",
+    checkbox_background_color_selected_dark="#0FCB8C",
+)
+try:
+    with open(LOGO_PATH, "r") as f:
+        svg_content = f.read()
+    encoded_svg = urllib.parse.quote(svg_content)
+    home_icon_data_uri = f"data:image/svg+xml,{encoded_svg}"
+except FileNotFoundError:
+    print(f"Warning: Home icon file not found at {LOGO_PATH}.")
+    home_icon_data_uri = "none"
+
+# --- This is the final CSS ---
+final_css = css + f"""
+/* --- Find the "Home" button and replace its text with an icon --- */
+.nav-holder nav a[href$="/"] {{
+    display: none !important;
+}}
+.nav-holder nav a[href*="/home"] {{
+    grid-row: 1 !important;
+    grid-column: 1 !important;
+    justify-self: start !important;
+    display: flex !important;
+    align-items: center !important;
+    justify-content: center !important;
+
+    /* 2. Hide the original "Home" text */
+    font-size: 0 !important;
+    text-indent: -9999px;
+
+    /* 3. Apply the icon as the background */
+    background-image: url("{home_icon_data_uri}") !important;
+    background-size: contain !important;
+    background-repeat: no-repeat !important;
+    background-position: center !important;
+
+    width: 240px !important;    
+    height: 50px !important;   
+    padding: 0 !important;
+    border: none !important;
+    outline: none !important;
+}}
+"""
+# --- Gradio App Definition ---
+demo = gr.Blocks(
+    theme=theme,
+    css=final_css,
+    head=scroll_script + redirect_script + tooltip_script + redirect_submission_on_close_script,
+    title="OpenHands Index",
+)
+with demo.route("Home", "/home"):
+    build_main_page()
+
+with demo.route("Literature Understanding", "/literature-understanding"):
+    build_lit_page()
+
+with demo.route("Code & Execution", "/code-execution"):
+    build_c_and_e_page()
+
+with demo.route("Data Analysis", "/data-analysis"):
+    build_data_analysis_page()
+
+with demo.route("End-to-End Discovery", "/discovery"):
+    build_e2e_page()
+
+with demo.route("About", "/about"):
+    build_about_page()
+
+with demo.route("🚀 Submit an Agent", "/submit"):
+    build_submission_page()
+# --- Scheduler and Launch
+def restart_space_job():
+    print("Scheduler: Attempting to restart space.")
+    try:
+        api.restart_space(repo_id=LEADERBOARD_PATH)
+        print("Scheduler: Space restart request sent.")
+    except Exception as e:
+        print(f"Scheduler: Error restarting space: {e}")
+    scheduler = BackgroundScheduler(timezone="UTC")
+    scheduler.add_job(restart_space_job, "interval", hours=1)
+    scheduler.start()
+
+
+# Launch the Gradio app
+if __name__ == "__main__":
+    if LOCAL_DEBUG:
+        print("Launching in LOCAL_DEBUG mode.")
+        demo.launch(debug=True, allowed_paths=["assets"], favicon_path="assets/favicon/favicon.ico")
+    else:
+        print("Launching in Space mode.")
+        # For Spaces, share=False is typical unless specific tunneling is needed.
+        # debug=True can be set to False for a "production" Space.
+        demo.launch(server_name="0.0.0.0", server_port=7860, debug=True, share=False, allowed_paths=["assets"], favicon_path="assets/favicon/favicon.ico")
+

c_and_e.py

@@ -0,0 +1,9 @@
+import gradio as gr
+from content import CODE_EXECUTION_DESCRIPTION
+from category_page_builder import build_category_page
+
+# Define the category for this page
+CATEGORY_NAME = "Code & Execution"
+
+def build_page():
+    build_category_page(CATEGORY_NAME, CODE_EXECUTION_DESCRIPTION)

category_page_builder.py

@@ -0,0 +1,105 @@
+import gradio as gr
+import pandas as pd
+
+# Import our UI factories and the data loader
+from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data, create_sub_navigation_bar
+CATEGORY_DIAGRAM_MAP = {
+    "Literature Understanding": "assets/literature-understanding.svg",
+    "Code & Execution": "assets/code-execution.svg",
+    "Data Analysis": "assets/data-analysis.svg",
+    "End-to-End Discovery": "assets/end-to-end-discovery.svg",
+}
+
+def build_category_page(CATEGORY_NAME, PAGE_DESCRIPTION):
+    with gr.Column(elem_id="page-content-wrapper"):
+        validation_df, validation_tag_map = get_full_leaderboard_data("validation")
+        test_df, test_tag_map = get_full_leaderboard_data("test")
+        with gr.Row(elem_id="intro-row"):
+
+            with gr.Column(scale=1):
+                gr.HTML(f'<h2>AstaBench {CATEGORY_NAME} Leaderboard <span style="font-weight: normal; color: inherit;">(Aggregate)</span></h2>', elem_id="main-header")
+                with gr.Column(elem_id="validation_nav_container", visible=False) as validation_nav_container:
+                    create_sub_navigation_bar(validation_tag_map, CATEGORY_NAME, validation=True)
+
+                with gr.Column(elem_id="test_nav_container", visible=True) as test_nav_container:
+                    create_sub_navigation_bar(test_tag_map, CATEGORY_NAME)
+
+                gr.Markdown(PAGE_DESCRIPTION, elem_id="intro-category-paragraph")
+
+            # --- The Right Column ---
+            with gr.Column(scale=1):
+                image_path = CATEGORY_DIAGRAM_MAP.get(CATEGORY_NAME)
+                if image_path:
+                    gr.Image(
+                        value=image_path,
+                        show_label=False,
+                        show_download_button=False,
+                        show_fullscreen_button=False,
+                        show_share_button=False,
+                        interactive=False,
+                        elem_id="diagram-image"
+                    )
+        # --- This page now has two main sections: Validation and Test ---
+        with gr.Tabs():
+            with gr.Tab("Results: Test Set") as test_tab:
+                # Repeat the process for the "test" split
+                if not test_df.empty:
+                    gr.Markdown("**Test Set** results are reserved for final assessment. This helps ensure that the agent generalizes well to unseen problems.")
+                    create_leaderboard_display(
+                        full_df=test_df,
+                        tag_map=test_tag_map,
+                        category_name=CATEGORY_NAME,
+                        split_name="test"
+                    )
+                    create_benchmark_details_display(
+                        full_df=test_df,
+                        tag_map=test_tag_map,
+                        category_name=CATEGORY_NAME,
+                        validation=False,
+                    )
+                else:
+                    gr.Markdown("No data available for test split.")
+            with gr.Tab("Results: Validation Set") as validation_tab:
+                # 1. Load all necessary data for the "validation" split ONCE.
+                if not validation_df.empty:
+                    gr.Markdown("**Validation Set** results are used during development to tune and compare agents before final testing.")
+                    # 2. Render the main category display using the loaded data.
+                    create_leaderboard_display(
+                        full_df=validation_df,
+                        tag_map=validation_tag_map,
+                        category_name=CATEGORY_NAME,
+                        split_name="validation"
+                    )
+
+                    # 3. Render the detailed breakdown for each benchmark in the category.
+                    create_benchmark_details_display(
+                        full_df=validation_df,
+                        tag_map=validation_tag_map,
+                        category_name=CATEGORY_NAME,
+                        validation=True,
+                    )
+                else:
+                    gr.Markdown("No data available for validation split.")
+
+
+        show_validation_js = """
+            () => {
+                document.getElementById('validation_nav_container').style.display = 'block';
+                document.getElementById('test_nav_container').style.display = 'none';
+                setTimeout(() => { window.dispatchEvent(new Event('resize')) }, 0);
+            }
+            """
+
+        # JavaScript to show the TEST nav, hide the VALIDATION nav, AND fix the plots.
+        show_test_js = """
+            () => {
+                document.getElementById('validation_nav_container').style.display = 'none';
+                document.getElementById('test_nav_container').style.display = 'block';
+            }
+            """
+
+        # Assign the pure JS functions to the select events. No Python `fn` is needed.
+        validation_tab.select(fn=None, inputs=None, outputs=None, js=show_validation_js)
+        test_tab.select(fn=None, inputs=None, outputs=None, js=show_test_js)
+
+    return validation_nav_container, test_nav_container

config.py

@@ -0,0 +1,22 @@
+import os
+
+LOCAL_DEBUG = not (os.environ.get("system") == "spaces")
+CONFIG_NAME = os.getenv("HF_CONFIG", "1.0.0-dev1") # This corresponds to 'config' in LeaderboardViewer
+IS_INTERNAL = os.environ.get("IS_INTERNAL", "false").lower() == "true"
+
+# OpenHands Index datasets
+CONTACT_DATASET = f"OpenHands/openhands-index-contact-info"
+
+if IS_INTERNAL:
+    # datasets backing the internal leaderboard
+    SUBMISSION_DATASET = f"OpenHands/openhands-index-internal-submissions"
+    RESULTS_DATASET = f"OpenHands/openhands-index-internal-results"
+    LEADERBOARD_PATH = f"OpenHands/openhands-index-internal-leaderboard"
+else:
+    # datasets backing the public leaderboard
+    SUBMISSION_DATASET = f"OpenHands/openhands-index-submissions"
+    RESULTS_DATASET = f"OpenHands/openhands-index-results"
+    LEADERBOARD_PATH = f"OpenHands/openhands-index"
+
+DATA_DIR = "/tmp/oh_index/data/" + CONFIG_NAME
+EXTRACTED_DATA_DIR = os.path.join(DATA_DIR, "extracted")

content.py

@@ -0,0 +1,934 @@
+import re
+
+def create_gradio_anchor_id(text: str, validation) -> str:
+    """
+    Replicates the ID format created by gr.Markdown(header_links=True).
+    Example: "Paper Finder Validation" -> "h-paper-finder-validation"
+    """
+    text = text.lower()
+    text = re.sub(r'\s+', '-', text) # Replace spaces with hyphens
+    text = re.sub(r'[^\w-]', '', text) # Remove non-word characters
+    if validation:
+        return f"h-{text}-leaderboard-1"
+    return f"h-{text}-leaderboard"
+
+
+TITLE = """<h1 align="left" id="space-title">AstaBench Leaderboard</h1>"""
+
+INTRO_PARAGRAPH = """
+<p>
+    <strong>AstaBench</strong> provides an aggregated view of agent performance and efficiency across all benchmarks in all four categories. We report:
+</p>
+
+<ul class="info-list">
+    <li>
+        <strong>Overall score:</strong> A macro-average of the four category-level average scores. Each category contributes equally, regardless of how many benchmarks it includes. This ensures fair comparisons across agents with different domain strengths.
+    </li>
+    <li>
+        <strong>Overall cost:</strong> A macro-average of the agent’s cost per problem across all categories, in USD. Each category contributes equally.
+    </li>
+</ul>
+
+<p>
+    This view is designed for quick comparison of general-purpose scientific agents. For more details on how we calculate scores and cost, please see the <a href="/about" style="color: #0FCB8C; text-decoration: underline;">About</a> Page.
+</p>
+"""
+SCATTER_DISCLAIMER = """
+**Note:** Agents without cost data are displayed to the right of the vertical divider line.
+"""
+PARETO_DISCLAIMER = """
+Agents names that are green are Pareto optimal, meaning they achieve the best performance for their cost. 
+"""
+LIT_DESCRIPTION = """
+The **Literature Understanding** category evaluates how well agents comprehend and interact with scientific literature—testing their ability to find research papers, assess citation quality, extract information from text, and more.
+<br><br>
+The scores shown below reflect performance aggregated across five distinct benchmarks, each targeting a different aspect of literature-based reasoning. 
+<br><br>
+For detailed results, use the links above to explore individual benchmarks.
+<br>
+"""
+CODE_EXECUTION_DESCRIPTION = """
+The **Code & Execution** category in AstaBench includes tasks that evaluate an agent’s ability to write, modify, and run code in realistic research scenarios. Unlike literature tasks—which only require read-only tools and can sometimes even be solved by a language model alone—these problems often require the agent to manipulate a machine environment with tools: reading input files, executing code, and writing outputs to specific files in the required format.
+<br><br>
+The scores in this category are aggregated from three distinct benchmarks, each targeting different facets of scientific coding and execution. Together, these benchmarks evaluate whether an agent can function as a hands-on scientific assistant—not just by reasoning about code, but by running it in real-world contexts.
+<br><br>
+For detailed results, use the links above to explore individual benchmark pages.
+<br>
+"""
+DATA_ANALYSIS_DESCRIPTION = """
+The **Data Analysis** category evaluates agents on their ability to analyze structured datasets and generate meaningful scientific hypotheses. It currently includes a single benchmark, DiscoveryBench, so the category-level scores are the same as the benchmark-level results.
+<br><br>
+As additional benchmarks are added in the future, this category will expand to cover a broader range of data-driven reasoning tasks across scientific domains.
+<br>
+"""
+DISCOVERY_DESCRIPTION = """
+The **End-to-End Discovery** category tests whether agents can carry out a complete scientific workflow, from task description to experiment design, code execution, results  analysis, and report writing. These tasks require agents to integrate multiple capabilities, producing not just answers but full research artifacts.
+<br><br>
+Scores in this category are aggregated from two benchmarks, providing the first standardized way to evaluate automated scientific discovery (ASD) agents across all stages of the research process. Use the links above to explore individual benchmark pages.
+<br>
+"""
+SUBMISSION_CONFIRMATION = """
+**Your agent has been submitted to AstaBench for evaluation.**
+<br><br>
+🙏 Thanks for contributing!
+<br><br>
+You'll receive a confirmation email from our team within 2 business days with next steps. We will reach out to you directly if further information is needed.
+<br><br>
+We appreciate your support in advancing scientific AI.
+"""
+
+# External URLs for benchmark descriptions
+SCHOLAR_QA_CS_URL = "https://www.semanticscholar.org/paper/OpenScholar%3A-Synthesizing-Scientific-Literature-LMs-Asai-He/b40df4b273f255b3cb5639e220c8ab7b1bdb313e"
+LITQA2_URL = "https://www.semanticscholar.org/paper/Language-agents-achieve-superhuman-synthesis-of-Skarlinski-Cox/fa5f9aa1cb6f97654ca8e6d279ceee1427a87e68"
+ARXIV_DIGESTABLES_URL = "https://www.semanticscholar.org/paper/ArxivDIGESTables%3A-Synthesizing-Scientific-into-Newman-Lee/c7face35e84f2cb04fb1600d54298799aa0ed189"
+SUPER_URL = "https://www.semanticscholar.org/paper/SUPER%3A-Evaluating-Agents-on-Setting-Up-and-Tasks-Bogin-Yang/053ef8299988680d47df36224bfccffc817472f1"
+CORE_BENCH_URL = "https://www.semanticscholar.org/paper/CORE-Bench%3A-Fostering-the-Credibility-of-Published-Siegel-Kapoor/4c913d59d150fe7581386b87dfd9f90448a9adee"
+DS1000_URL = "https://arxiv.org/abs/2211.11501"
+DISCOVERY_BENCH_URL = "https://www.semanticscholar.org/paper/DiscoveryBench%3A-Towards-Data-Driven-Discovery-with-Majumder-Surana/48c83799530dc523ee01e6c1c40ad577d5c10a16"
+
+# Helper function to create external links
+def external_link(url, text, is_s2_url=False):
+    url = f"{url}?utm_source=asta_leaderboard" if is_s2_url else url
+    return f"<a href='{url}' target='_blank' rel='noopener noreferrer'>{text}</a>"
+
+def internal_leaderboard_link(text, validation):
+    anchor_id = create_gradio_anchor_id(text, validation)
+    return f"<a href='#{anchor_id}'>{text}</a>"
+
+# Function to get benchmark descriptions with validation flag
+def get_benchmark_description(benchmark_name, validation):
+    descriptions = {
+    'PaperFindingBench': (
+        "PaperFindingBench assesses an agent's ability to locate sets of papers based on a natural language "
+        "description that may involve both the papers' content and metadata, such as the author or publication year."
+    ),
+    'LitQA2-FullText-Search': (
+        f"A version of {internal_leaderboard_link('LitQA2-FullText', validation)} that isolates the retrieval aspect of the task. "
+        f"This benchmark features the same multi-choice questions as {internal_leaderboard_link('LitQA2-FullText', validation)}, but the agent is not evaluated on answering the actual question "
+        "but rather on providing a ranked list of papers in which the answer is likely to be found."
+    ),
+    'ScholarQA-CS2': (
+        "ScholarQA-CS2 assesses long-form model responses to literature review questions in the domain of computer science. "
+        "Answers are expected to be comprehensive reports, such as those produced by deep research systems. "
+        f"This benchmark advances on the previously released {external_link(SCHOLAR_QA_CS_URL, 'ScholarQA-CS', is_s2_url=True)} "
+        "by using queries from real-world usage, and introducing new evaluation methods for coverage and precision "
+        "of both the report text and its citations."
+    ),
+    'LitQA2-FullText': (
+        f"{external_link(LITQA2_URL, 'LitQA2', is_s2_url=True)}, a benchmark introduced by FutureHouse, gauges a model's ability to answer questions that require document retrieval from the scientific literature. "
+        "It consists of multiple-choice questions that necessitate finding a unique paper and analyzing its detailed full text to spot precise information; these questions cannot be answered from a paper’s abstract. "
+        "While the original version of the benchmark provided for each question the title of the paper in which the answer can be found, it did not specify the overall collection to search over. In our version, "
+        "we search over the index we provide as part of the Asta standard toolset. The “-FullText” suffix indicates we consider only the subset of LitQA2 questions for which "
+        "the full-text version of the answering paper is open source and available in our index."
+    ),
+    'ArxivDIGESTables-Clean': (
+        f"{external_link(ARXIV_DIGESTABLES_URL, 'ArxivDIGESTables', is_s2_url=True)} assesses the ability of models to construct literature review tables, i.e., tables whose rows are papers and whose columns constitute a set of "
+        "aspects used to compare and contrast the papers. The goal is to construct such tables given a set of related papers and a table caption describing the user's goal. Generated tables are evaluated by "
+        "comparing them to actual tables published in ArXiv papers. The “-Clean” suffix indicates a curated subset of ArxivDIGESTables which drops tables that are either trivial or impossible to reconstruct from full-texts."
+    ),
+    'SUPER-Expert': (
+        "SUPER-Expert evaluates the capability of models in setting up and executing tasks from low-resource "
+        "research repositories—centralized databases containing research data and related materials. "
+        f"The \"-Expert\" split indicates the name of the most challenging split in the {external_link(SUPER_URL, 'original SUPER benchmark', is_s2_url=True)} "
+        "that involves solving reproduction tasks from scratch and without any intermediate hints or details "
+        "about the important landmarks involved in each task."
+    ),
+    'CORE-Bench-Hard': (
+        "Core-Bench-Hard tests computational reproducibility, a task involving reproducing the results of a study "
+        "using provided code and data. It consists of both language-only and vision-language challenges across "
+        "multiple difficulty levels. "
+        f"The \"-Hard\" split refers to the name of the most challenging split in the original {external_link(CORE_BENCH_URL, 'Core-bench benchmark', is_s2_url=True)} "
+        "where only a README file is provided with no instructions or an auxiliary Dockerfile."
+    ),
+    'DS-1000': (
+        "DS-1000 is an established code generation benchmark containing Python data science coding questions "
+        "originally sourced from StackOverflow. It's designed to reflect an array of diverse, realistic, and "
+        "practical use cases and directly involves many of the Python libraries commonly used in data science "
+        f"and machine learning research. We split the original {external_link(DS1000_URL, 'dataset')} "
+        "into 100 validation and 900 test problems."
+    ),
+    'DiscoveryBench': (
+        "DiscoveryBench is the first comprehensive benchmark to formalize the multi-step process of data-driven "
+        "analysis and discovery (i.e., data loading, transformation, statistical analysis, and modeling). "
+        f"Originally introduced {external_link(DISCOVERY_BENCH_URL, 'here', is_s2_url=True)}, it is designed to systematically "
+        "evaluate how well current LLMs can replicate or reproduce published scientific findings across diverse "
+        "domains, including social science, biology, history, and more."
+    ),
+    'E2E-Bench': (
+        "E2E-Bench is the \"decathlon\" of AI-assisted research. It measures whether a system can run the entire "
+        "research pipeline, starting with an initial task description, to designing and performing (software) "
+        "experiments, to analyzing and writing up the results."
+    ),
+    'E2E-Bench-Hard': (
+        f"E2E-Bench-Hard is a more challenging variant of {internal_leaderboard_link('E2E-Bench', validation)}. Tasks are generated using the HypER system, "
+        "which identifies research trends and proposes new, underexplored problems. Unlike the regular version, "
+        "these tasks are not simplified or curated for accessibility; they are reviewed only for feasibility. "
+        "This version is intended to test whether systems can handle more complex and less-structured research "
+        f"scenarios, following the same end-to-end process as {internal_leaderboard_link('E2E-Bench', validation)}."
+    )
+    }
+    
+    return descriptions.get(benchmark_name, "")
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@article{asta-bench,
+    title={AstaBench},
+    author={AstaBench folks},
+    year={2025},
+    eprint={TBD.TBD},
+    archivePrefix={arXiv},
+    primaryClass={cs.AI},
+    secondaryClass={cs.CL}
+}"""
+
+LEGAL_DISCLAIMER_TEXT = """
+<h2>Terms and Conditions</h2>
+<p>
+    The Allen Institute for Artificial Intelligence (Ai2) maintains this repository for agent evaluation submissions to AstaBench. To keep AstaBench fair and auditable, all evaluation logs and associated submission files will be made publicly available. This includes your benchmark inputs, model output responses, and other data and information related to your submission as needed to verify the results.
+</p>
+<br>
+<p>
+    Your submissions to AstaBench will be posted, scored, and ranked on the leaderboard at <a href="https://huggingface.co/spaces/allenai/asta-bench-leaderboard" target="_blank" rel="noopener noreferrer">https://huggingface.co/spaces/allenai/asta-bench-leaderboard</a>. You agree you have the rights to the materials you submit and that you will not share any personal, sensitive, proprietary, or confidential information.
+</p>
+"""
+
+def format_error(msg):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{msg}</p>"
+
+
+def format_warning(msg):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{msg}</p>"
+
+
+def format_log(msg):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>"
+
+
+def hyperlink(link_url: str, text: str = "🔗") -> str:
+    if not link_url or not isinstance(link_url, str):
+        return str(text) # Or simply "" if link_url is bad
+    return f'<a target="_blank" href="{link_url}">{text}</a>'
+
+
+def hf_uri_to_web_url(uri: str) -> str:
+    """
+    Convert a Hugging Face-style URI like:
+        hf://datasets/{namespace}/{repo}/{path...}
+    into a public web URL:
+        https://huggingface.co/datasets/{namespace}/{repo}/tree/main/{path...}
+    """
+    prefix = "hf://datasets/"
+    if not uri.startswith(prefix):
+        raise ValueError("URI must start with 'hf://datasets/'")
+
+    parts = uri[len(prefix) :].split("/", 2)
+    if len(parts) < 3:
+        raise ValueError("Expected format: hf://datasets/{namespace}/{repo}/{path...}")
+
+    namespace, repo, path = parts
+    return f"https://huggingface.co/datasets/{namespace}/{repo}/tree/main/{path}"
+
+
+css = """
+/* CSS Color Variables using Gradio theme */
+:root {
+    --color-primary-green: var(--primary-900); /* #0FCB8C */
+    --color-primary-pink: var(--secondary-900); /* #f0529c */
+    --color-neutral-light: var(--neutral-200); /* #C9C9C3 */
+    --color-background-light: var(--neutral-50); /* #FAF2E9 */
+    --color-background-dark: var(--neutral-900); /* #032629 */
+    --color-text-light: var(--neutral-50); /* #FAF2E9 */
+}
+
+/* This makes space for the huggingface header bar which must shown on HF spaces. */
+/* FIXME Media queries don't seem to survive rendering. */
+/* @media (min-width: 768px) { ... } */
+gradio-app {
+    padding-top: 65px;
+}
+
+/* Global Styles */
+h2 {
+    overflow: hidden;
+}
+
+#intro-paragraph {
+    font-size: 18px;
+    max-width: 90%;
+    padding-left: 35px;
+    margin-top: 20px;
+}
+
+#intro-paragraph p,
+#intro-paragraph li {
+    font-size: 16px; 
+    line-height: 1.8; 
+}
+
+#intro-paragraph ul {
+    margin-top: 20px;
+    margin-bottom: 20px;
+}
+
+#diagram-image {
+    height: 100%;
+}
+
+#diagram-image img {
+    width: 100%;
+    height: 100%;
+    object-fit: cover; 
+}
+#intro-category-paragraph {
+    font-size: 18px;
+    max-width: 90%;
+    margin-top: 20px;
+}
+
+#intro-category-paragraph p,
+#intro-category-paragraph li {
+    font-size: 16px; 
+    line-height: 1.8; 
+}
+
+#intro-category-paragraph ul {
+    margin-top: 20px;
+    margin-bottom: 20px;
+}
+
+#about-content {
+    font-size: 18px;
+    max-width: 60%;
+    padding-left: 25px;
+}
+#category-intro {
+    font-size: 18px;
+    max-width: 60%;
+}
+#logo-image { 
+    margin: 0;
+    margin-bottom: 30px; 
+    justify-content: flex-start;        
+    max-width: 250px;       
+    height: auto;           
+}
+#page-content-wrapper{
+    padding-left: 25px;
+}
+.table-component{
+    height: auto !important;
+    max-height: none !important;
+}
+.table-wrap {
+    max-height: none !important;
+    height: auto !important;
+    overflow-y: visible !important;
+}
+/* --- New Rules for Table Density --- */
+table.gr-table th, table.gr-table td {
+    padding: 4px 4px !important; 
+    width: 1%;
+    white-space: nowrap;
+}
+table.svelte-1e98i6s td {
+    vertical-align: top !important;
+}
+table.gr-table {
+    font-size: 14px !important;
+}
+.html-container {
+    padding-top: 0 !important;
+}
+#scatter-disclaimer {
+        overflow: visible !important;
+}
+#pareto-disclaimer {
+    color: #f0529c !important;
+}
+thead.svelte-1e98i6s th {
+    background: white !important;
+}
+.dark thead.svelte-1e98i6s th {
+    background: #091a1a !important;
+}
+.cell-wrap.svelte-v1pjjd {
+    font-family: 'Manrope';
+    }
+nav.svelte-ti537g.svelte-ti537g {
+    justify-content: flex-start;
+}
+.nav-holder {
+    padding-left: 20px !important;
+}
+#legend-markdown span {
+    margin-right: 15px !important; 
+}
+#leaderboard-accordion .label-wrap {
+    font-size: 1.4rem !important; 
+    z-index: 10 !important;
+    position: relative !important;
+}
+.dark #leaderboard-accordion .label-wrap {
+    color: #0FCB8C !important; 
+}
+.dark block.svelte-1svsvh2 {
+    background: #032629 !important;
+}
+.padding.svelte-phx28p {
+    padding: 0 !important;
+}
+.sub-nav-bar-container {
+    display: flex !important;
+    flex-wrap: wrap !important; 
+    align-items: center !important; 
+    gap: 10px !important;
+}
+.dark .primary-link-button {
+    color: var(--color-primary-green);
+}
+.primary-link-button {
+    background: none;
+    border: none;
+    padding: 0;
+    margin: 0;
+    font-family: inherit;
+    font-size: 16px;
+    color: var(--color-primary-pink);
+    text-decoration: none;
+    cursor: pointer;
+    white-space: nowrap;
+}
+.primary-link-button:hover {
+    text-decoration: underline;
+}
+.sub-nav-label {
+    font-weight: bold;
+    font-size: 16px;
+    display: flex;
+    align-items: center;
+}
+.wrap-header-df th span{
+    white-space: normal !important;
+    word-break: normal !important;
+    overflow-wrap: break-word !important;
+    line-height: 1.2 !important;
+    vertical-align: top !important;
+    font-size: 12px !important;
+    font-family: 'Manrope';
+}
+.wrap-header-df th {
+    height: auto !important;
+}
+.wrap-header-df .cell-wrap img {
+    width: 16px;
+    height: 16px;
+    vertical-align: middle;
+}
+#legend-markdown img {
+    width: 16px;
+    height: 16px;
+    vertical-align: middle;
+}
+/*------ Global tooltip styles ------*/
+.tooltip-icon {
+    display: inline-block;
+    cursor: help;
+    position: relative;
+}
+.tooltip-icon::after {
+    content: attr(data-tooltip);
+    position: absolute;
+    bottom: 125%;
+    background-color: #105257;
+    color: #fff;
+    padding: 10px;
+    border-radius: 4px;
+    font-size: 12px;
+    opacity: 0;
+    transition: opacity 0.2s;
+    white-space: pre-line;
+    width: max-content;
+    text-align: left;
+    pointer-events: none;
+    max-width: 300px;
+    left: 50%;
+    transform: translateX(-50%);
+    z-index: 1000;
+}
+@media (max-width: 768px) {
+    .tooltip-icon::after {
+        max-width: 250px;
+    }
+}
+.tooltip-icon:hover::after {
+    opacity: 1;
+}
+/*------ Openness label tooltip styles ------*/
+.styler,
+#openness-label-html,
+#agent-tooling-label-html {
+    overflow: visible !important;
+}
+/*------ Table cell tooltip styles ------*/
+.wrap.default.full,
+span.wrap[tabindex="0"][role="button"][data-editable="false"] {
+  overflow: visible !important;
+}
+
+.cell-tooltip-icon::after {
+    height: fit-content;
+    top: 125%;
+}
+/*------ Table column description tooltip styles ------*/
+#legend-markdown,
+#leaderboard-accordion {
+    overflow: visible !important;
+}
+
+/* --- inside table tooltips --- */
+.native-tooltip-icon {
+    cursor: help;
+    text-decoration: underline dotted 1px;
+}
+/* Main Nav bar styling */
+.nav-holder nav {
+    display: grid !important;
+    grid-template-columns: auto auto auto auto auto 1fr auto auto !important;
+    gap: 10px 20px !important; /* Vertical and horizontal spacing */
+    width: 100% !important;
+    align-items: center;
+}
+.nav-holder nav a[href*="about"] {
+    grid-row: 1 !important;
+    grid-column: 7 !important;
+}
+.nav-holder nav a[href*="submit"] {
+    grid-row: 1 !important;
+    grid-column: 8 !important;
+    white-space: nowrap !important;
+}
+/* Divider line between header and category nav */
+.nav-holder nav::after {
+    content: ''; /* Required for pseudo-elements to appear */
+    background-color: #C9C9C3;
+    height: 1px; 
+    grid-row: 2 !important;
+    grid-column: 1 / -1 !important;
+}
+
+/* Horizontal scrolling for navigation */
+.nav-holder nav {
+    overflow-x: auto;
+    scrollbar-width: none;
+    -ms-overflow-style: none;
+}
+.nav-holder nav::-webkit-scrollbar {
+    display: none;
+}
+
+/* Category navigation buttons in row 3 */
+.nav-holder nav a[href*="literature-understanding"],
+.nav-holder nav a[href*="code-execution"],
+.nav-holder nav a[href*="data-analysis"],
+.nav-holder nav a[href*="discovery"] {
+    grid-row: 3 !important;
+    justify-self: center !important;
+    width: fit-content !important;
+    white-space: nowrap;
+    flex-shrink: 0;
+}
+
+.nav-holder nav a[href*="literature-understanding"] { grid-column: 1 !important; }
+.nav-holder nav a[href*="code-execution"] { grid-column: 2 !important; }
+.nav-holder nav a[href*="data-analysis"] { grid-column: 3 !important; }
+.nav-holder nav a[href*="discovery"] { grid-column: 4 !important; }
+
+/* Navigation hover styles */
+.nav-holder nav a[href*="about"]:hover,
+.nav-holder nav a[href*="submit"]:hover,
+.nav-holder nav a[href*="literature-understanding"]:hover,
+.nav-holder nav a[href*="code-execution"]:hover,
+.nav-holder nav a[href*="data-analysis"]:hover,
+.nav-holder nav a[href*="discovery"]:hover {
+    background-color: #FDF9F4;
+}
+
+.dark .nav-holder nav a[href*="about"]:hover,
+.dark .nav-holder nav a[href*="submit"]:hover,
+.dark .nav-holder nav a[href*="literature-understanding"]:hover,
+.dark .nav-holder nav a[href*="code-execution"]:hover,
+.dark .nav-holder nav a[href*="data-analysis"]:hover,
+.dark .nav-holder nav a[href*="discovery"]:hover {
+    background-color: #1C3A3C;
+}
+.benchmark-main-subtitle{
+    color: var(--color-primary-green);
+    overflow: hidden;
+    padding-top: 120px;
+}
+.benchmark-title{
+    color: var(--color-primary-pink);
+    margin-top: 50px;
+        font-size: 20px;
+}
+.dark .benchmark-title{
+    color: var(--color-primary-green);
+}
+.benchmark-description {
+    margin: 20px 0;
+    max-width: 800px;
+}
+/*------ Submission Page CSS ------*/
+#submission-modal .modal-container,
+#success-modal .modal-container {
+    height: auto;
+    max-width: 600px;
+}
+
+#submission-modal-content,
+#success-modal .submission-modal-content { 
+    padding: 20px;
+    background-color: inherit;
+    border-radius: 8px;
+    text-align: center;
+}
+
+#submission-modal-content p,
+#success-modal .submission-modal-content p {
+    font-size: 16px;
+}
+
+#legal-modal-content {
+    padding: 30px;
+    background-color: inherit;
+    border-radius: 8px;
+    text-align: left;
+    font-size: 14px;
+}
+
+#legal-modal-content h2 {
+    text-align: center;
+}
+#legal-modal-content button {
+    width: fit-content;
+}
+.spinner-container {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    padding: 30px;
+}
+    
+.spinner {
+    width: 50px;
+    height: 50px;
+    border: 5px solid #dee2e6;
+    border-top: 5px solid #007bff;
+    border-radius: 50%;
+    animation: spin 1s linear infinite;
+    margin-bottom: 20px;
+}
+    
+@keyframes spin {
+    0% { transform: rotate(0deg); }
+    100% { transform: rotate(360deg); }
+}
+    
+#submission-page-container {
+    max-width: 800px;
+    margin: 0 auto;
+}
+
+#submission-file-label {
+    padding: 10px;
+}
+
+#submission-button {
+    max-width: fit-content;
+    font-size: 14px;
+}
+
+.custom-form-group {
+    border: 1px solid #000 !important; 
+    border-radius: 4px !important;
+    padding: 24px !important;
+    overflow: visible !important;    
+}
+
+#openness-label-html,
+#agent-tooling-label-html,
+#agent-info-label-html,
+#submitter-info-label-html,
+#username-label-html,
+#email-label-html,
+#role-label-html  {
+    padding-left: 12px;
+}
+
+.form-label {
+    margin: 4px 0px 0px 6px;
+}
+
+.form-label-fieldset {
+    padding-top: 10px !important;
+}
+
+#agent-tooling-label-html {
+    padding-top: 6px;
+}
+
+.custom-form-group,
+.styler {
+    background: none;
+}
+
+#feedback-button {
+    display: inline-block;
+    background-color: #345d60;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    padding: 15px 20px;
+    font-size: 16px;
+    cursor: pointer;
+    transition: all 0.3s ease;
+    text-decoration: none;
+}
+
+#feedback-button:hover {
+    background-color: #5d888b;
+    transform: translateY(-2px);
+    box-shadow: 0 6px 12px rgba(0,0,0,0.3);
+}
+.dark #main-header h2 {
+    color: #0fcb8c; 
+}
+#main-header h2 {
+    color: #f0529c;
+}
+
+/* --- New HTML-Based Tooltip Styles --- */
+.tooltip-icon-legend {
+    position: relative;
+    cursor: help;
+    display: inline-block;
+}
+
+/* The HTML pop-up card tooltips.*/
+.tooltip-card {
+    /* Hiding mechanism */
+    opacity: 0;
+    visibility: hidden;
+    transition: opacity 0.2s;
+    pointer-events: none;
+    /* Card appearance */
+    position: fixed;
+    z-index: 1000;
+    background-color: #083c40;
+    color: #e5e7eb;
+    border-radius: 12px;
+    padding: 15px;
+    width: max-content;
+    max-width: 400px;
+    text-align: left;
+}
+.tooltip-card.visible {
+    opacity: 1;
+    visibility: visible;
+} 
+.tooltip-card h3 {
+    font-size: 18px; 
+    color: #fff; 
+    margin-top: 0; 
+    margin-bottom: 12px;
+}
+.tooltip-card .tooltip-description {
+    margin-bottom: 20px; 
+    line-height: 1.3;
+}
+.tooltip-card .tooltip-items-container {
+    display: flex; 
+    flex-direction: column; 
+    gap: 10px;
+}
+.tooltip-card .tooltip-legend-item {
+    display: flex; 
+    align-items: 
+    flex-start; 
+    gap: 10px;
+}
+.tooltip-card .tooltip-legend-item img {
+    width: 20px; 
+    height: 20px; 
+    margin-top: 2px;
+}
+.tooltip-card .tooltip-legend-item div {
+    display: flex; 
+    flex-direction: column;
+}
+.tooltip-card .tooltip-legend-item strong {
+    font-weight: 600; 
+    color: #fff;
+}
+.tooltip-card .tooltip-legend-item span {
+    font-size: 13px; 
+    line-height: 1.3;
+}
+.tooltip-sub-list {
+    list-style-type: '• '; 
+    padding-left: 18px;         
+    font-size: 13px;
+    line-height: 1.3;  
+    display: flex;
+    flex-direction: column;   
+} 
+.table-legend-item {  
+    display: flex; 
+    align-items: center; 
+    white-space: nowrap; 
+    margin-top: 8px; 
+    flex-wrap: wrap;
+}
+
+/* About Page CSS */
+#about-page-content-wrapper {
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 800px; 
+    padding: 0 24px;
+    display: flex;
+    flex-direction: column; 
+    gap: 40px; 
+    margin-top: 40px;
+    opacity: 85%; 
+    margin-bottom: 60px;
+}
+.link-buttons-container {
+    display: flex;
+    flex-wrap: wrap; /* Allows buttons to stack on very narrow screens */
+    gap: 16px;     
+    margin-top: 16px;
+}
+.link-button {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    flex-grow: 1; 
+    background-color: #083c40; 
+    padding: 16px 20px;
+    font-weight: 600;
+    border-radius: 12px;
+    text-decoration: none; 
+    transition: background-color 0.2s ease-in-out;
+}
+.link-button:hover {
+    background-color: #0a4c52; 
+}
+.external-link-icon {
+    font-size: 20px;
+    line-height: 1;
+    margin-left: 12px;
+}
+
+#leaderboard-accordion table {
+    width: auto !important;
+    margin-right: auto !important;
+}
+.info-list {
+    padding-left: 20px;
+}
+
+/* Smooth scrolling for the entire page */
+html {
+    scroll-behavior: smooth;
+}
+/* Home Page Styling */
+.diagram-placeholder {
+    width: 100%;
+    height: 100%; 
+    min-height: 250px; 
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    background-color: #FAF2E9; 
+    color: #F0529C;          
+    border-radius: 8px;
+    font-size: 14px;
+    text-align: center;
+}
+/* 2. Responsive behavior for smaller screens */
+@media (max-width: 900px) {
+    #intro-row {
+        flex-direction: column;
+    }
+}
+/* Plot legend styles */
+.plot-legend-container {
+    min-height: 572px;
+    background-color: #fff;
+    padding: 24px 32px;
+    border: 1px solid black;
+    border-radius: 4px;
+}
+
+.dark .plot-legend-container {
+    background: rgba(250, 242, 233, 0.1);
+    border-color: rgb(159, 234, 209);
+}
+
+#plot-legend-logo {
+    margin-bottom: 24px;
+}
+
+#plot-legend-logo img {
+    height: 19px;
+}
+
+.plot-legend-category-heading {
+    font-size: 16px;
+    font-weight: 700;    
+}
+
+.plot-legend-item {
+    display: flex;      
+    margin-top: 8px;
+}
+
+
+.plot-legend-item-text .description {
+    color: #888;
+    font-size: 12px;
+}
+
+.plot-legend-item-svg {
+    margin-top: 3px;
+    width: 14px;
+    height: 14px;
+    margin-right: 8px;
+}
+
+.plot-legend-tooling-svg {
+    height: 16px;
+    width: 16px;
+    margin-top: 2px;
+}
+
+#plot-legend-item-pareto-svg {
+    width: 18px;
+    height: 18px;
+    margin-right: 2px;
+}
+h3 .header-link-icon {
+    font-size: 12px;
+    vertical-align: text-top;
+    margin-left: 6px;
+    text-decoration: none;
+}
+
+/* Targets all "overall stats" columns in the main leaderboard for each category */
+#main-leaderboard td:nth-child(6) .prose,
+#main-leaderboard td:nth-child(7) .prose {
+    font-weight: 700 !important;
+}
+"""