sync from github

backend-cli.py

@@ -473,6 +473,7 @@ if __name__ == "__main__":
         precisions = args.precision.split(",")
         print(f"debug_model_names: {debug_model_names}, debug_task_name: {debug_task_name}, precisions: {precisions}")
         task_lst = TASKS_HARNESS.copy()
+        RESULTS_REPO = DEBUG_RESULTS_REPO
         for precision in precisions:
             for debug_model_name in debug_model_names:
                 for task in task_lst:

open-moe-llm-leaderboard-gh/.gitignore

@@ -1,166 +0,0 @@
-# LM eval
-eval-queue-bk/
-eval-results-bk/
-eval-queue/
-eval-results/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/

open-moe-llm-leaderboard-gh/Dockerfile

@@ -1,8 +0,0 @@
-# Use specific python image
-FROM registry.hf.space/sparse-generative-ai-open-moe-llm-leaderboard:latest
-
-RUN pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ moe-infinity --no-cache-dir
-# To fix pydantic version
-RUN pip install pydantic==2.6.4 --no-cache-dir
-# To fix selfcheck (selfchatgpt) dataset missing
-RUN python -m spacy download en

open-moe-llm-leaderboard-gh/Makefile

@@ -1,13 +0,0 @@
-.PHONY: style format
-
-
-style:
-	python -m black --line-length 119 .
-	python -m isort .
-	ruff check --fix .
-
-
-quality:
-	python -m black --check --line-length 119 .
-	python -m isort --check-only .
-	ruff check .

open-moe-llm-leaderboard-gh/README.md

@@ -1,85 +0,0 @@
----
-title: OPEN-MOE-LLM-LEADERBOARD
-emoji: 🔥
-colorFrom: green
-colorTo: indigo
-sdk: gradio
-sdk_version: 4.9.0
-app_file: app.py
-pinned: true
-license: apache-2.0
-fullWidth: true
-tags:
-  - leaderboard
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
-
-# Contributing to Open-MOE-LLM-Leaderboard
-
-Thank you for your interest in contributing to the Open-MOE-LLM-Leaderboard project! We welcome contributions from everyone. Below you'll find guidance on how to set up your development environment, understand our architecture, and contribute effectively. If you have any questions or wish to discuss your contributions, please reach out to Yao Fu via email at [Y.Fu@ed.ac.uk](mailto:y.fu@ed.ac.uk).
-
-## What We're Looking For in Contributions
-
-We are looking for contributions in several key areas to enhance the Open-MOE-LLM-Leaderboard project:
-
-1. **General Bug Fixes/Reports**: We welcome reports of any bugs found in the frontend interface or backend, as well as fixes for these issues.
-
-2. **Adding New Tasks (Benchmark Datasets)**: If you have ideas for new benchmark datasets that could be added, your contributions would be greatly appreciated.
-
-3. **Supporting New Inference Frameworks**: Expanding our project to support new inference frameworks is crucial for our growth. If you can contribute in this area, please reach out.
-
-4. **Testing More Models**: To make our leaderboard as comprehensive as possible, we need to test a wide range of models. Contributions in this area are highly valuable.
-
-Documentation is currently of lower priority, but if you have thoughts or suggestions, please feel free to raise them.
-
-Your contributions are crucial to the success and improvement of the Open-MOE-LLM-Leaderboard project. We look forward to collaborating with you.
-
-
-## Development Setup
-
-To start contributing, set up your development environment as follows:
-
-```bash
-conda create -n leaderboard python=3.10
-conda activate leaderboard
-pip install -r requirements.txt
-pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ moe-infinity
-pip install pydantic==2.6.4 # Resolves a dependency conflict with moe-infinity
-python -m spacy download en # Required for selfcheckgpt
-```
-
-## Architecture Overview
-
-The Open-MOE-LLM-Leaderboard project uses the following architecture:
-
-- **User Interface (Gradio)** ->upload-> **HuggingFace Dataset (Request)** ->download-> **Backend GPU Server** ->upload-> **HuggingFace Dataset (Result)** ->download-> **User Interface (Gradio)**
-
-In brief:
-1. Users submit model benchmarking requests through the Gradio interface ([app.py](./app.py)). These requests are then recorded in a HuggingFace dataset ([sparse-generative-ai/requests](https://huggingface.co/datasets/sparse-generative-ai/requests)).
-2. The backend ([backend-cli.py](./backend-cli.py)), running on a GPU server, processes these requests, performs the benchmarking tasks, and uploads the results to another HuggingFace dataset ([sparse-generative-ai/results](https://huggingface.co/datasets/sparse-generative-ai/results)).
-3. Finally, the Gradio interface retrieves and displays these results to the users.
-
-## Running the Gradio Interface
-
-To launch the Gradio interface, execute:
-
-```bash
-python app.py
-```
-
-Then, open your browser and navigate to http://127.0.0.1:7860.
-
-## Running the Backend
-
-To start the backend process, use:
-
-```bash
-python backend-cli.py --debug
-```
-
-For additional details, please consult the [backend-cli.py](./backend-cli.py) script.
-
----
-
-We look forward to your contributions and are here to help guide you through the process. Thank you for supporting the Open-MOE-LLM-Leaderboard project!

open-moe-llm-leaderboard-gh/app.py

@@ -1,495 +0,0 @@
-#!/usr/bin/env python
-import os
-import datetime
-import socket
-import base64
-from threading import Thread
-
-import gradio as gr
-import pandas as pd
-import time
-from apscheduler.schedulers.background import BackgroundScheduler
-
-from huggingface_hub import snapshot_download
-
-from src.display.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    LLM_BENCHMARKS_DETAILS,
-    FAQ_TEXT,
-    TITLE,
-    ACKNOWLEDGEMENT_TEXT,
-)
-
-from src.display.css_html_js import custom_css
-
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    TYPES,
-    AutoEvalColumn,
-    ModelType,
-    InferenceFramework,
-    fields,
-    WeightType,
-    Precision,
-    GPUType
-)
-
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, \
-    QUEUE_REPO, REPO_ID, RESULTS_REPO, DEBUG_QUEUE_REPO, DEBUG_RESULTS_REPO
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-from src.utils import get_dataset_summary_table
-
-def get_args():
-    import argparse
-
-    parser = argparse.ArgumentParser(description="Run the LLM Leaderboard")
-    parser.add_argument("--debug", action="store_true", help="Run in debug mode")
-    return parser.parse_args()
-
-args = get_args()
-if args.debug:
-    print("Running in debug mode")
-    QUEUE_REPO = DEBUG_QUEUE_REPO
-    RESULTS_REPO = DEBUG_RESULTS_REPO
-
-def ui_snapshot_download(repo_id, local_dir, repo_type, tqdm_class, etag_timeout):
-    try:
-        print(local_dir)
-        snapshot_download(
-            repo_id=repo_id, local_dir=local_dir, repo_type=repo_type, tqdm_class=tqdm_class, etag_timeout=etag_timeout
-        )
-    except Exception as e:
-        restart_space()
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
-
-
-def init_space():
-    dataset_df = get_dataset_summary_table(file_path="blog/Hallucination-Leaderboard-Summary.csv")
-
-    if socket.gethostname() not in {"neuromancer"}:
-        # sync model_type with open-llm-leaderboard
-        ui_snapshot_download(
-            repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-        )
-        ui_snapshot_download(
-            repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-        )
-    raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, "", COLS, BENCHMARK_COLS)
-
-    finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(
-        EVAL_REQUESTS_PATH, EVAL_COLS
-    )
-    return dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
-    
-    
-def add_benchmark_columns(shown_columns):
-    benchmark_columns = []
-    for benchmark in BENCHMARK_COLS:
-        if benchmark in shown_columns:
-            for c in COLS:
-                if benchmark in c and benchmark != c:
-                    benchmark_columns.append(c)
-    return benchmark_columns
-
-
-# Searching and filtering
-def update_table(
-    hidden_df: pd.DataFrame, columns: list, type_query: list, precision_query: list, size_query: list, query: str
-):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query)
-    filtered_df = filter_queries(query, filtered_df)
-    benchmark_columns = add_benchmark_columns(columns)
-    df = select_columns(filtered_df, columns + benchmark_columns)
-    return df
-
-
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
-
-
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-    # always_here_cols = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name]
-
-    always_here_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-    dummy_col = [AutoEvalColumn.dummy.name]
-
-    # We use COLS to maintain sorting
-    filtered_df = df[
-        # always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]
-        always_here_cols
-        + [c for c in COLS if c in df.columns and c in columns]
-        + dummy_col
-    ]
-    return filtered_df
-
-
-def filter_queries(query: str, filtered_df: pd.DataFrame):
-    final_df = []
-    if query != "":
-        queries = [q.strip() for q in query.split(";")]
-        for _q in queries:
-            _q = _q.strip()
-            if _q != "":
-                temp_filtered_df = search_table(filtered_df, _q)
-                if len(temp_filtered_df) > 0:
-                    final_df.append(temp_filtered_df)
-        if len(final_df) > 0:
-            filtered_df = pd.concat(final_df)
-            subset = [AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
-            filtered_df = filtered_df.drop_duplicates(subset=subset)
-    return filtered_df
-
-
-def filter_models(df: pd.DataFrame, type_query: list, size_query: list, precision_query: list) -> pd.DataFrame:
-    # Show all models
-    filtered_df = df
-
-    type_emoji = [t[0] for t in type_query]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-    filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-    # numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-    # params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    # mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-    # filtered_df = filtered_df.loc[mask]
-
-    return filtered_df
-
-shown_columns = None
-dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
-leaderboard_df = original_df.copy()
-
-# def update_leaderboard_table():
-#     global leaderboard_df, shown_columns
-#     print("Updating leaderboard table")
-#     return leaderboard_df[
-#                 [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-#                 + shown_columns.value
-#                 + [AutoEvalColumn.dummy.name]
-#             ] if not leaderboard_df.empty else leaderboard_df
-        
-
-# def update_hidden_leaderboard_table():
-#     global original_df
-#     return original_df[COLS] if original_df.empty is False else original_df
-
-# def update_dataset_table():
-#     global dataset_df
-#     return dataset_df
-
-# def update_finish_table():
-#     global finished_eval_queue_df
-#     return finished_eval_queue_df
-
-# def update_running_table():
-#     global running_eval_queue_df
-#     return running_eval_queue_df
-
-# def update_pending_table():
-#     global pending_eval_queue_df
-#     return pending_eval_queue_df
-
-# def update_finish_num():
-#     global finished_eval_queue_df
-#     return len(finished_eval_queue_df)
-
-# def update_running_num():
-#     global running_eval_queue_df
-#     return len(running_eval_queue_df)
-
-# def update_pending_num():
-#     global pending_eval_queue_df
-#     return len(pending_eval_queue_df)
-
-# triggered only once at startup => read query parameter if it exists
-def load_query(request: gr.Request):
-    query = request.query_params.get("query") or ""
-    return query
-
-
-def get_image_html(url, image_path):
-    with open(image_path, "rb") as image_file:
-        encoded_string = base64.b64encode(image_file.read()).decode()
-    return f'<a href="{url}" target="_blank"><img src="data:image/jpg;base64,{encoded_string}" alt="NetMind.AI Logo" style="width:100pt;"></a>'
-
-
-# Prepare the HTML content with the image
-image_html = get_image_html("https://netmind.ai/home", "./src/display/imgs/Netmind.AI_LOGO.jpg")
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-    gr.HTML(ACKNOWLEDGEMENT_TEXT.format(image_html=image_html))
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("open-moe-llm-leaderboard", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Model search (separate multiple queries with `;`)",
-                            show_label=False,
-                            elem_id="search-bar"
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if not c.hidden and not c.never_hidden and not c.dummy
-                            ],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-
-                with gr.Column(min_width=320):
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Inference frameworks",
-                        choices=[t.to_str() for t in InferenceFramework],
-                        value=[t.to_str() for t in InferenceFramework],
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
-
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
-
-                    # filter_columns_size = gr.CheckboxGroup(
-                    #     label="Model sizes (in billions of parameters)",
-                    #     choices=list(NUMERIC_INTERVALS.keys()),
-                    #     value=list(NUMERIC_INTERVALS.keys()),
-                    #     interactive=True,
-                    #     elem_id="filter-columns-size",
-                    # )
-
-            # breakpoint()
-            benchmark_columns = add_benchmark_columns(shown_columns.value)
-            leaderboard_table = gr.components.Dataframe(
-                value=(
-                    leaderboard_df[
-                        [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-                        + shown_columns.value
-                        + benchmark_columns
-                        + [AutoEvalColumn.dummy.name]
-                    ]
-                    if leaderboard_df.empty is False
-                    else leaderboard_df
-                ),
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value + benchmark_columns,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )  # column_widths=["2%", "20%"]
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS] if original_df.empty is False else original_df,
-                headers=COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    search_bar,
-                ],
-                leaderboard_table
-            )
-
-            # Check query parameter once at startup and update search bar
-            demo.load(load_query, inputs=[], outputs=[search_bar])
-
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        filter_columns_type,
-                        filter_columns_precision,
-                        filter_columns_size,
-                        search_bar,
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
-
-        with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-            dataset_table = gr.components.Dataframe(
-                value=dataset_df,
-                headers=list(dataset_df.columns),
-                datatype=["str", "markdown", "str", "str", "str"],
-                elem_id="dataset-table",
-                interactive=False,
-                visible=True,
-                column_widths=["15%", "20%"],
-            )
-
-            gr.Markdown(LLM_BENCHMARKS_DETAILS, elem_classes="markdown-text")
-            gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("Submit a model ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(f"✅ Finished Evaluations ({len(finished_eval_queue_df)})", open=False):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5
-                            )
-
-                    with gr.Accordion(f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})", open=False):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5
-                            )
-
-                    with gr.Accordion(f"⏳ Scheduled Evaluation Queue ({len(pending_eval_queue_df)})", open=False):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5
-                            )
-
-            with gr.Row():
-                gr.Markdown("# Submit your model here", elem_classes="markdown-text")
-
-            with gr.Row():
-                inference_framework = gr.Dropdown(
-                    choices=[t.to_str() for t in InferenceFramework],
-                    label="Inference framework",
-                    multiselect=False,
-                    value=None,
-                    interactive=True,
-                )
-                
-                gpu_type = gr.Dropdown(
-                    choices=[t.to_str() for t in GPUType],
-                    label="GPU type",
-                    multiselect=False,
-                    value="NVIDIA-A100-PCIe-80GB",
-                    interactive=True,
-                )
-                
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float32",
-                        interactive=True,
-                    )
-
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            debug = gr.Checkbox(value=args.debug, label="Debug", visible=False)
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    private,
-                    weight_type,
-                    model_type,
-                    inference_framework,
-                    debug,
-                    gpu_type
-                ],
-                submission_result,
-            )
-
-    with gr.Row():
-        with gr.Accordion("Citing this leaderboard", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-
-scheduler = BackgroundScheduler()
-
-scheduler.add_job(restart_space, "interval", hours=6)
-
-def launch_backend():
-    import subprocess
-    from src.backend.envs import DEVICE
-
-    if DEVICE not in {"cpu"}:
-        _ = subprocess.run(["python", "backend-cli.py"])
-
-# Thread(target=periodic_init, daemon=True).start()
-# scheduler.add_job(launch_backend, "interval", seconds=120)
-if __name__ == "__main__":
-    scheduler.start()
-    demo.queue(default_concurrency_limit=40).launch()
-    

open-moe-llm-leaderboard-gh/backend-cli.py

@@ -1,539 +0,0 @@
-#!/usr/bin/env python
-
-import os
-import json
-import argparse
-
-import socket
-import random
-import threading
-from datetime import datetime
-
-from src.backend.run_eval_suite import run_evaluation
-from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
-from src.backend.sort_queue import sort_models_by_priority
-from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, Task
-from src.backend.manage_requests import EvalRequest
-from src.leaderboard.read_evals import EvalResult
-
-from src.envs import QUEUE_REPO, RESULTS_REPO, API, DEBUG_QUEUE_REPO, DEBUG_RESULTS_REPO
-from src.utils import my_snapshot_download, analyze_gpu_stats, parse_nvidia_smi, monitor_gpus, get_gpu_details
-
-from src.leaderboard.read_evals import get_raw_eval_results
-
-from typing import Optional
-import GPUtil
-import time
-
-import pprint
-import logging
-
-from lm_eval.filters.extraction import RegexFilter
-
-
-# Configure the root logger
-logging.basicConfig(
-    format="%(asctime)s,%(msecs)03d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s",
-    datefmt="%Y-%m-%d:%H:%M:%S",
-    level=logging.WARNING,
-)
-
-# Get the 'lm-eval' logger from the third-party library
-eval_logger = logging.getLogger("lm-eval")
-
-# Explicitly set the level for 'lm-eval' logger to WARNING
-eval_logger.setLevel(logging.WARNING)
-
-def tuple_input_decorator(func):
-    def wrapper(self, resps, docs):
-        stripped_resps = [[resp_data[0] for resp_data in group] for group in resps]
-
-        filtered_resps = func(self, stripped_resps, docs)
-        
-        combined_resps = []
-        for original_group, new_group in zip(resps, filtered_resps):
-            combined_group = [(new_resp,) + rest_of_data[1:] for new_resp, rest_of_data in zip(new_group, original_group)]
-            combined_resps.append(combined_group)
-
-        return combined_resps
-    return wrapper
-
-
-def my_set_eval_request(api, eval_request, set_to_status, hf_repo, local_dir):
-    for i in range(10):
-        try:
-            set_eval_request(
-                api=api, eval_request=eval_request, set_to_status=set_to_status, hf_repo=hf_repo, local_dir=local_dir
-            )
-            return
-        except Exception as e:
-            print(f"Error setting eval request to {set_to_status}: {e}. Retrying in 60 seconds")
-            time.sleep(60)
-    return
-
-
-logging.getLogger("openai").setLevel(logging.WARNING)
-
-logging.basicConfig(level=logging.ERROR)
-pp = pprint.PrettyPrinter(width=80)
-
-PENDING_STATUS = "PENDING"
-RUNNING_STATUS = "RUNNING"
-FINISHED_STATUS = "FINISHED"
-FAILED_STATUS = "FAILED"
-
-TASKS_HARNESS = [task.value for task in Tasks]
-
-
-my_snapshot_download(
-    repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-)
-my_snapshot_download(
-    repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-)
-
-
-def sanity_checks():
-    print(f"Device: {DEVICE}")
-
-    # pull the eval dataset from the hub and parse any eval requests
-    # check completed evals and set them to finished
-    my_snapshot_download(
-        repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-    )
-    check_completed_evals(
-        api=API,
-        checked_status=RUNNING_STATUS,
-        completed_status=FINISHED_STATUS,
-        failed_status=FAILED_STATUS,
-        hf_repo=QUEUE_REPO,
-        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-        hf_repo_results=RESULTS_REPO,
-        local_dir_results=EVAL_RESULTS_PATH_BACKEND,
-    )
-    return
-
-
-def request_to_result_name(request: EvalRequest) -> str:
-    # Request: EvalRequest(model='meta-llama/Llama-2-13b-hf', private=False, status='FINISHED',
-    # json_filepath='./eval-queue-bk/meta-llama/Llama-2-13b-hf_eval_request_False_False_False.json',
-    # weight_type='Original', model_type='pretrained', precision='float32', base_model='', revision='main',
-    # submitted_time='2023-09-09T10:52:17Z', likes=389, params=13.016, license='?')
-    #
-    # EvalResult(eval_name='meta-llama_Llama-2-13b-hf_float32', full_model='meta-llama/Llama-2-13b-hf',
-    # org='meta-llama', model='Llama-2-13b-hf', revision='main',
-    # results={'nq_open': 33.739612188365655, 'triviaqa': 74.12505572893447},
-    # precision=<Precision.float32: ModelDetails(name='float32', symbol='')>,
-    # model_type=<ModelType.PT: ModelDetails(name='pretrained', symbol='🟢')>,
-    # weight_type=<WeightType.Original: ModelDetails(name='Original', symbol='')>,
-    # architecture='LlamaForCausalLM', license='?', likes=389, num_params=13.016, date='2023-09-09T10:52:17Z', still_on_hub=True)
-    #
-    org_and_model = request.model.split("/", 1)
-    if len(org_and_model) == 1:
-        model = org_and_model[0]
-        res = f"{model}_{request.precision}"
-    else:
-        org = org_and_model[0]
-        model = org_and_model[1]
-        res = f"{org}_{model}_{request.precision}"
-    return res
-
-
-def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[int] = None) -> dict:
-    batch_size = 1
-    batch_size = eval_request.batch_size
-
-    init_gpu_info = analyze_gpu_stats(parse_nvidia_smi())
-    # if init_gpu_info['Mem(M)'] > 500:
-    #     assert False, f"This machine is not empty: {init_gpu_info}"
-    gpu_stats_list = []
-    stop_event = threading.Event()
-    monitor_thread = threading.Thread(target=monitor_gpus, args=(stop_event, 5, gpu_stats_list))
-    monitor_thread.start()
-    
-    original_apply = RegexFilter.apply
-    if task.benchmark in ["gsm8k", "gsm8k_cot", "gsm8k_cot_self_consistency", "gsm8k_custom"]:
-        RegexFilter.apply = tuple_input_decorator(RegexFilter.apply)
-    else:
-        RegexFilter.apply = original_apply
-
-    try:
-        results = run_evaluation(
-            eval_request=eval_request,
-            task_names=[task.benchmark],
-            num_fewshot=task.num_fewshot,
-            batch_size=batch_size,
-            device=DEVICE,
-            use_cache=None,
-            limit=limit,
-        )
-    except RuntimeError as e:
-        if "No executable batch size found" in str(e):
-            batch_size = 1
-            results = run_evaluation(
-                eval_request=eval_request,
-                task_names=[task.benchmark],
-                num_fewshot=task.num_fewshot,
-                batch_size=batch_size,
-                device=DEVICE,
-                use_cache=None,
-                limit=limit,
-            )
-        else:
-            raise
-
-    # print("RESULTS", results)
-    stop_event.set()
-    monitor_thread.join()
-    gpu_info = analyze_gpu_stats(gpu_stats_list)
-    for task_name in results['results'].keys():
-        for key, value in gpu_info.items():
-            if "GPU" not in key:
-                results['results'][task_name][f"{key},none"] = int(value)
-            else:
-                results['results'][task_name][f"{key},none"] = value
-
-        results['results'][task_name]['batch_size,none'] = batch_size
-        results['results'][task_name]['precision,none'] = eval_request.precision
-    print(f"gpu_stats_list: {gpu_stats_list}")
-    print("GPU Usage:", gpu_info)
-
-    dumped = json.dumps(results, indent=2, default=lambda o: "<not serializable>")
-    # print(dumped)
-
-    output_path = os.path.join(
-        EVAL_RESULTS_PATH_BACKEND, *eval_request.model.split("/"), f"results_{datetime.now()}.json"
-    )
-    os.makedirs(os.path.dirname(output_path), exist_ok=True)
-    with open(output_path, "w") as f:
-        f.write(dumped)
-
-    my_snapshot_download(
-        repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-    )
-    API.upload_file(
-        path_or_fileobj=output_path,
-        path_in_repo=f"{eval_request.model}/results_{datetime.now()}.json",
-        repo_id=RESULTS_REPO,
-        repo_type="dataset",
-    )
-    
-    RegexFilter.apply = original_apply
-    return results
-
-
-def process_finished_requests(thr: int, hard_task_lst: Optional[list[str]] = None) -> bool:
-    sanity_checks()
-
-    current_finished_status = [FINISHED_STATUS, FAILED_STATUS]
-
-    # Get all eval request that are FINISHED, if you want to run other evals, change this parameter
-    eval_requests: list[EvalRequest] = get_eval_requests(
-        job_status=current_finished_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND
-    )
-    # Sort the evals by priority (first submitted, first run)
-    eval_requests: list[EvalRequest] = sort_models_by_priority(api=API, models=eval_requests)
-
-    random.shuffle(eval_requests)
-
-    eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH_BACKEND, EVAL_REQUESTS_PATH_BACKEND)
-
-    result_name_to_request = {request_to_result_name(r): r for r in eval_requests}
-    result_name_to_result = {r.eval_name: r for r in eval_results}
-
-    for eval_request in eval_requests:
-        if eval_request.likes >= thr:
-            result_name: str = request_to_result_name(eval_request)
-
-            # Check the corresponding result
-            eval_result: Optional[EvalResult] = (
-                result_name_to_result[result_name] if result_name in result_name_to_result else None
-            )
-
-            # breakpoint()
-
-            task_lst = TASKS_HARNESS.copy()
-            random.shuffle(task_lst)
-
-            # Iterate over tasks and, if we do not have results for a task, run the relevant evaluations
-            for task in task_lst:
-                task_name = task.benchmark
-
-                do_run_task = False
-                if hard_task_lst is None or any(ss in task_name for ss in hard_task_lst):
-                    do_run_task = True
-
-                if (eval_result is None or task_name not in eval_result.results) and do_run_task:
-                    eval_request: EvalRequest = result_name_to_request[result_name]
-
-                    my_snapshot_download(
-                        repo_id=QUEUE_REPO,
-                        revision="main",
-                        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-                        repo_type="dataset",
-                        max_workers=60,
-                    )
-                    my_set_eval_request(
-                        api=API,
-                        eval_request=eval_request,
-                        set_to_status=RUNNING_STATUS,
-                        hf_repo=QUEUE_REPO,
-                        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-                    )
-
-                    results = process_evaluation(task, eval_request)
-
-                    my_snapshot_download(
-                        repo_id=QUEUE_REPO,
-                        revision="main",
-                        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-                        repo_type="dataset",
-                        max_workers=60,
-                    )
-                    my_set_eval_request(
-                        api=API,
-                        eval_request=eval_request,
-                        set_to_status=FINISHED_STATUS,
-                        hf_repo=QUEUE_REPO,
-                        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-                    )
-
-                    return True
-
-    return False
-
-
-def maybe_refresh_results(thr: int, hard_task_lst: Optional[list[str]] = None) -> bool:
-    sanity_checks()
-
-    current_finished_status = [PENDING_STATUS, FINISHED_STATUS, FAILED_STATUS]
-
-    # Get all eval request that are FINISHED, if you want to run other evals, change this parameter
-    eval_requests: list[EvalRequest] = get_eval_requests(
-        job_status=current_finished_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND
-    )
-    # Sort the evals by priority (first submitted, first run)
-    eval_requests: list[EvalRequest] = sort_models_by_priority(api=API, models=eval_requests)
-
-    random.shuffle(eval_requests)
-
-    eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH_BACKEND, EVAL_REQUESTS_PATH_BACKEND)
-
-    result_name_to_request = {request_to_result_name(r): r for r in eval_requests}
-    result_name_to_result = {r.eval_name: r for r in eval_results}
-
-    for eval_request in eval_requests:
-        if eval_request.likes >= thr:
-            result_name: str = request_to_result_name(eval_request)
-
-            # Check the corresponding result
-            eval_result: Optional[EvalResult] = (
-                result_name_to_result[result_name] if result_name in result_name_to_result else None
-            )
-
-            task_lst = TASKS_HARNESS.copy()
-            random.shuffle(task_lst)
-
-            # Iterate over tasks and, if we do not have results for a task, run the relevant evaluations
-            for task in task_lst:
-                task_name = task.benchmark
-
-                do_run_task = False
-                if hard_task_lst is None or any(ss in task_name for ss in hard_task_lst):
-                    do_run_task = True
-
-                task_lst = ["nq", "trivia", "tqa", "self"]
-                if (
-                    eval_result is None
-                    or do_run_task
-                    or task_name not in eval_result.results
-                    or any(ss in task_name for ss in task_lst)
-                ):
-                    eval_request: EvalRequest = result_name_to_request[result_name]
-
-                    my_snapshot_download(
-                        repo_id=QUEUE_REPO,
-                        revision="main",
-                        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-                        repo_type="dataset",
-                        max_workers=60,
-                    )
-                    my_set_eval_request(
-                        api=API,
-                        eval_request=eval_request,
-                        set_to_status=RUNNING_STATUS,
-                        hf_repo=QUEUE_REPO,
-                        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-                    )
-
-                    results = process_evaluation(task, eval_request)
-
-                    my_snapshot_download(
-                        repo_id=QUEUE_REPO,
-                        revision="main",
-                        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-                        repo_type="dataset",
-                        max_workers=60,
-                    )
-                    my_set_eval_request(
-                        api=API,
-                        eval_request=eval_request,
-                        set_to_status=FINISHED_STATUS,
-                        hf_repo=QUEUE_REPO,
-                        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-                    )
-
-                    return True
-
-    return False
-
-def process_pending_requests() -> bool:
-    sanity_checks()
-    print("Processing pending requests")
-    current_pending_status = [PENDING_STATUS]
-
-    # Get all eval request that are PENDING, if you want to run other evals, change this parameter
-    eval_requests = get_eval_requests(
-        job_status=current_pending_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND
-    )
-    # Sort the evals by priority (first submitted, first run)
-    eval_requests = sort_models_by_priority(api=API, models=eval_requests)
-
-    random.shuffle(eval_requests)
-
-    print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
-
-    if len(eval_requests) == 0:
-        return False
-
-    eval_request = eval_requests[0]
-    pp.pprint(eval_request)
-    
-    gpu_type = eval_request.gpu_type
-    curr_gpu_type = get_gpu_details()
-    if gpu_type != curr_gpu_type:
-        print(f"GPU type mismatch: {gpu_type} vs {curr_gpu_type}")
-        return False
-
-    my_snapshot_download(
-        repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-    )
-    my_set_eval_request(
-        api=API,
-        eval_request=eval_request,
-        set_to_status=RUNNING_STATUS,
-        hf_repo=QUEUE_REPO,
-        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-    )
-
-    task_lst = TASKS_HARNESS.copy()
-    random.shuffle(task_lst)
-
-    for task in task_lst:
-        results = process_evaluation(task, eval_request)
-
-    my_snapshot_download(
-        repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-    )
-    my_set_eval_request(
-        api=API,
-        eval_request=eval_request,
-        set_to_status=FINISHED_STATUS,
-        hf_repo=QUEUE_REPO,
-        local_dir=EVAL_REQUESTS_PATH_BACKEND,
-    )
-
-    return True
-
-
-def get_args():
-    parser = argparse.ArgumentParser(description="Run the backend")
-    parser.add_argument("--debug", action="store_true", help="Run in debug mode")
-    # debug parameters
-    parser.add_argument("--task", type=str, default="selfcheckgpt,mmlu, gsm8k", help="Task to debug")
-    parser.add_argument("--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1,mistralai/Mixtral-8x7B-v0.1", help="Model to debug")
-    parser.add_argument("--precision", type=str, default="float32,float16,8bit,4bit", help="Precision to debug")
-    parser.add_argument("--inference-framework", type=str, default="hf-chat", help="Inference framework to debug")
-    parser.add_argument("--limit", type=int, default=None, help="Limit for the number of samples")
-    parser.add_argument("--gpu-type", type=str, default="NVIDIA-A100-PCIe-80GB", 
-                        help="GPU type. NVIDIA-A100-PCIe-80GB; NVIDIA-RTX-A5000-24GB; NVIDIA-H100-PCIe-80GB")
-    parser.add_argument("--debug_repo", action="store_true", help="Use debug repo")
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = get_args()
-    local_debug = args.debug
-    # debug specific task by ping
-    if local_debug and not args.debug_repo:
-        # debug_model_names = [args.model]  # Use model from arguments
-        # debug_task_name = [args.task]  # Use task from arguments
-        debug_model_names = args.model.split(",")
-        debug_task_name = args.task.split(",")
-        precisions = args.precision.split(",")
-        print(f"debug_model_names: {debug_model_names}, debug_task_name: {debug_task_name}, precisions: {precisions}")
-        task_lst = TASKS_HARNESS.copy()
-        RESULTS_REPO = DEBUG_RESULTS_REPO
-        for precision in precisions:
-            for debug_model_name in debug_model_names:
-                for task in task_lst:
-                    task_name = task.benchmark
-                    if task_name not in debug_task_name:
-                        continue
-                    # try:
-                    eval_request = EvalRequest(
-                        model=debug_model_name,
-                        private=False,
-                        status="",
-                        json_filepath="",
-                        precision=precision,  # Use precision from arguments
-                        inference_framework=args.inference_framework,  # Use inference framework from arguments
-                        gpu_type=args.gpu_type
-                    )
-                    curr_gpu_type = get_gpu_details()
-                    if eval_request.gpu_type != curr_gpu_type:
-                        print(f"GPU type mismatch: {eval_request.gpu_type} vs {curr_gpu_type}")
-                        raise Exception("GPU type mismatch")
-                    results = process_evaluation(task, eval_request, limit=args.limit)
-                    # except Exception as e:
-                    #     print(f"debug running error: {e}")
-    elif local_debug and args.debug_repo:
-        QUEUE_REPO = DEBUG_QUEUE_REPO
-        RESULTS_REPO = DEBUG_RESULTS_REPO
-        while True:
-            res = False
-            # if random.randint(0, 10) == 0:
-            res = process_pending_requests()
-            print(f"waiting for 60 seconds")
-            time.sleep(60)
-            # if res is False:
-            #     if random.randint(0, 5) == 0:
-            #         res = maybe_refresh_results(100)
-            #     else:
-            #         res = process_finished_requests(100)
-            # time.sleep(60)
-            # if res is False:
-            #     if random.randint(0, 5) == 0:
-            #         res = maybe_refresh_results(0)
-            #     else:
-            #         res = process_finished_requests(0)
-    elif not local_debug and not args.debug_repo:
-        while True:
-           res = False
-           # if random.randint(0, 10) == 0:
-           res = process_pending_requests()
-           print(f"waiting for 60 seconds")
-           time.sleep(60)
-           # if res is False:
-           #     if random.randint(0, 5) == 0:
-           #         res = maybe_refresh_results(100)
-           #     else:
-           #         res = process_finished_requests(100)
-           # time.sleep(60)
-           # if res is False:
-           #     if random.randint(0, 5) == 0:
-           #         res = maybe_refresh_results(0)
-           #     else:
-           #         res = process_finished_requests(0)
-    else:
-        raise Exception("Cannot use debug_repo without local debug flag")

open-moe-llm-leaderboard-gh/cli/analysis-cli.py

@@ -1,353 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import json
-import pickle
-
-import numpy as np
-
-import pandas as pd
-import seaborn as sns
-import matplotlib.pyplot as plt
-
-from scipy.cluster.hierarchy import linkage
-
-from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Task
-
-from src.envs import QUEUE_REPO, RESULTS_REPO, API
-from src.utils import my_snapshot_download
-
-
-def is_float(string):
-    try:
-        float(string)
-        return True
-    except ValueError:
-        return False
-
-
-def find_json_files(json_path):
-    res = []
-    for root, dirs, files in os.walk(json_path):
-        for file in files:
-            if file.endswith(".json"):
-                res.append(os.path.join(root, file))
-    return res
-
-
-def sanitise_metric(name: str) -> str:
-    res = name
-    res = res.replace("prompt_level_strict_acc", "Prompt-Level Accuracy")
-    res = res.replace("acc", "Accuracy")
-    res = res.replace("exact_match", "EM")
-    res = res.replace("avg-selfcheckgpt", "AVG")
-    res = res.replace("max-selfcheckgpt", "MAX")
-    res = res.replace("rouge", "ROUGE-")
-    res = res.replace("bertscore_precision", "BERT-P")
-    res = res.replace("exact", "EM")
-    res = res.replace("HasAns_EM", "HasAns")
-    res = res.replace("NoAns_EM", "NoAns")
-    res = res.replace("em", "EM")
-    return res
-
-
-def sanitise_dataset(name: str) -> str:
-    res = name
-    res = res.replace("tqa8", "TriviaQA (8-shot)")
-    res = res.replace("nq8", "NQ (8-shot)")
-    res = res.replace("nq_open", "NQ (64-shot)")
-    res = res.replace("triviaqa", "TriviaQA (64-shot)")
-    res = res.replace("truthfulqa", "TruthfulQA")
-    res = res.replace("ifeval", "IFEval")
-    res = res.replace("selfcheckgpt", "SelfCheckGPT")
-    res = res.replace("truefalse_cieacf", "True-False")
-    res = res.replace("mc", "MC")
-    res = res.replace("race", "RACE")
-    res = res.replace("squad", "SQuAD")
-    res = res.replace("memo-trap", "MemoTrap")
-    res = res.replace("cnndm", "CNN/DM")
-    res = res.replace("xsum", "XSum")
-    res = res.replace("qa", "QA")
-    res = res.replace("summarization", "Summarization")
-    res = res.replace("dialogue", "Dialog")
-    res = res.replace("halueval", "HaluEval")
-    res = res.replace("_v2", "")
-    res = res.replace("_", " ")
-    return res
-
-
-cache_file = "data_map_cache.pkl"
-
-
-def load_data_map_from_cache(cache_file):
-    if os.path.exists(cache_file):
-        with open(cache_file, "rb") as f:
-            return pickle.load(f)
-    else:
-        return None
-
-
-def save_data_map_to_cache(data_map, cache_file):
-    with open(cache_file, "wb") as f:
-        pickle.dump(data_map, f)
-
-
-# Try to load the data_map from the cache file
-data_map = load_data_map_from_cache(cache_file)
-
-
-if data_map is None:
-    my_snapshot_download(
-        repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-    )
-    my_snapshot_download(
-        repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-    )
-
-    result_path_lst = find_json_files(EVAL_RESULTS_PATH_BACKEND)
-    request_path_lst = find_json_files(EVAL_REQUESTS_PATH_BACKEND)
-
-    model_name_to_model_map = {}
-
-    for path in request_path_lst:
-        with open(path, "r") as f:
-            data = json.load(f)
-        model_name_to_model_map[data["model"]] = data
-
-    model_dataset_metric_to_result_map = {}
-
-    # data_map[model_name][(dataset_name, sanitised_metric_name)] = value
-    data_map = {}
-
-    for path in result_path_lst:
-        with open(path, "r") as f:
-            data = json.load(f)
-        model_name = data["config"]["model_name"]
-        for dataset_name, results_dict in data["results"].items():
-            for metric_name, value in results_dict.items():
-
-                if model_name_to_model_map[model_name]["likes"] > 128:
-
-                    to_add = True
-
-                    if "f1" in metric_name:
-                        to_add = False
-
-                    if "stderr" in metric_name:
-                        to_add = False
-
-                    if "memo-trap_v2" in dataset_name:
-                        to_add = False
-
-                    if "faithdial" in dataset_name:
-                        to_add = False
-
-                    if "truthfulqa_gen" in dataset_name:
-                        to_add = False
-
-                    if "bertscore" in metric_name:
-                        if "precision" not in metric_name:
-                            to_add = False
-
-                    if "halueval" in dataset_name:
-                        if "acc" not in metric_name:
-                            to_add = False
-
-                    if "ifeval" in dataset_name:
-                        if "prompt_level_strict_acc" not in metric_name:
-                            to_add = False
-
-                    if "squad" in dataset_name:
-                        # to_add = False
-                        if "best_exact" in metric_name:
-                            to_add = False
-
-                    if "fever" in dataset_name:
-                        to_add = False
-
-                    if ("xsum" in dataset_name or "cnn" in dataset_name) and "v2" not in dataset_name:
-                        to_add = False
-
-                    if isinstance(value, str):
-                        if is_float(value):
-                            value = float(value)
-                        else:
-                            to_add = False
-
-                    if to_add:
-                        if "rouge" in metric_name:
-                            value /= 100.0
-
-                        if "squad" in dataset_name:
-                            value /= 100.0
-
-                        sanitised_metric_name = metric_name
-                        if "," in sanitised_metric_name:
-                            sanitised_metric_name = sanitised_metric_name.split(",")[0]
-                        sanitised_metric_name = sanitise_metric(sanitised_metric_name)
-                        sanitised_dataset_name = sanitise_dataset(dataset_name)
-
-                        model_dataset_metric_to_result_map[
-                            (model_name, sanitised_dataset_name, sanitised_metric_name)
-                        ] = value
-
-                        if model_name not in data_map:
-                            data_map[model_name] = {}
-                        data_map[model_name][(sanitised_dataset_name, sanitised_metric_name)] = value
-
-                        print(
-                            "model_name",
-                            model_name,
-                            "dataset_name",
-                            sanitised_dataset_name,
-                            "metric_name",
-                            sanitised_metric_name,
-                            "value",
-                            value,
-                        )
-
-    save_data_map_to_cache(data_map, cache_file)
-
-model_name_lst = [m for m in data_map.keys()]
-
-nb_max_metrics = max(len(data_map[model_name]) for model_name in model_name_lst)
-
-for model_name in model_name_lst:
-    if len(data_map[model_name]) < nb_max_metrics - 5:
-        del data_map[model_name]
-
-plot_type_lst = ["all", "summ", "qa", "instr", "detect", "rc"]
-
-for plot_type in plot_type_lst:
-
-    data_map_v2 = {}
-    for model_name in data_map.keys():
-        for dataset_metric in data_map[model_name].keys():
-            if dataset_metric not in data_map_v2:
-                data_map_v2[dataset_metric] = {}
-
-            if plot_type in {"all"}:
-                to_add = True
-                if "ROUGE" in dataset_metric[1] and "ROUGE-L" not in dataset_metric[1]:
-                    to_add = False
-                if "SQuAD" in dataset_metric[0] and "EM" not in dataset_metric[1]:
-                    to_add = False
-                if "SelfCheckGPT" in dataset_metric[0] and "MAX" not in dataset_metric[1]:
-                    to_add = False
-                if "64-shot" in dataset_metric[0]:
-                    to_add = False
-                if to_add is True:
-                    data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
-            elif plot_type in {"summ"}:
-                if "CNN" in dataset_metric[0] or "XSum" in dataset_metric[0]:
-                    data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
-            elif plot_type in {"qa"}:
-                if "TriviaQA" in dataset_metric[0] or "NQ" in dataset_metric[0] or "TruthfulQA" in dataset_metric[0]:
-                    data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
-            elif plot_type in {"instr"}:
-                if "MemoTrap" in dataset_metric[0] or "IFEval" in dataset_metric[0]:
-                    data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
-            elif plot_type in {"detect"}:
-                if "HaluEval" in dataset_metric[0] or "SelfCheck" in dataset_metric[0]:
-                    data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
-            elif plot_type in {"rc"}:
-                if "RACE" in dataset_metric[0] or "SQuAD" in dataset_metric[0]:
-                    data_map_v2[dataset_metric][model_name] = data_map[model_name][dataset_metric]
-            else:
-                assert False, f"Unknown plot type: {plot_type}"
-
-    # df = pd.DataFrame.from_dict(data_map, orient='index')   # Invert the y-axis (rows)
-    df = pd.DataFrame.from_dict(data_map_v2, orient="index")  # Invert the y-axis (rows)
-    df.index = [", ".join(map(str, idx)) for idx in df.index]
-
-    o_df = df.copy(deep=True)
-
-    # breakpoint()
-
-    print(df)
-
-    # Check for NaN or infinite values and replace them
-    df.replace([np.inf, -np.inf], np.nan, inplace=True)  # Replace infinities with NaN
-    df.fillna(0, inplace=True)  # Replace NaN with 0 (or use another imputation strategy)
-
-    from sklearn.preprocessing import MinMaxScaler
-
-    # scaler = MinMaxScaler()
-    # df = pd.DataFrame(scaler.fit_transform(df), index=df.index, columns=df.columns)
-
-    # Calculate dimensions based on the DataFrame size
-    cell_height = 1.0  # Height of each cell in inches
-    cell_width = 1.0  # Width of each cell in inches
-
-    n_rows = len(df.index)  # Datasets and Metrics
-    n_cols = len(df.columns)  # Models
-
-    # Calculate figure size dynamically
-    fig_width = cell_width * n_cols + 0
-    fig_height = cell_height * n_rows + 0
-
-    col_cluster = True
-    row_cluster = True
-
-    sns.set_context("notebook", font_scale=1.3)
-
-    dendrogram_ratio = (0.1, 0.1)
-
-    if plot_type in {"detect"}:
-        fig_width = cell_width * n_cols - 2
-        fig_height = cell_height * n_rows + 5.2
-        dendrogram_ratio = (0.1, 0.2)
-
-    if plot_type in {"instr"}:
-        fig_width = cell_width * n_cols - 2
-        fig_height = cell_height * n_rows + 5.2
-        dendrogram_ratio = (0.1, 0.4)
-
-    if plot_type in {"qa"}:
-        fig_width = cell_width * n_cols - 2
-        fig_height = cell_height * n_rows + 4
-        dendrogram_ratio = (0.1, 0.2)
-
-    if plot_type in {"summ"}:
-        fig_width = cell_width * n_cols - 2
-        fig_height = cell_height * n_rows + 2.0
-        dendrogram_ratio = (0.1, 0.1)
-        row_cluster = False
-
-    if plot_type in {"rc"}:
-        fig_width = cell_width * n_cols - 2
-        fig_height = cell_height * n_rows + 5.2
-        dendrogram_ratio = (0.1, 0.4)
-
-    print("figsize", (fig_width, fig_height))
-
-    o_df.to_json(f"plots/clustermap_{plot_type}.json", orient="split")
-
-    print(f"Generating the clustermaps for {plot_type}")
-
-    for cmap in [None, "coolwarm", "viridis"]:
-        fig = sns.clustermap(
-            df,
-            method="ward",
-            metric="euclidean",
-            cmap=cmap,
-            figsize=(fig_width, fig_height),  # figsize=(24, 16),
-            annot=True,
-            mask=o_df.isnull(),
-            dendrogram_ratio=dendrogram_ratio,
-            fmt=".2f",
-            col_cluster=col_cluster,
-            row_cluster=row_cluster,
-        )
-
-        # Adjust the size of the cells (less wide)
-        plt.setp(fig.ax_heatmap.get_yticklabels(), rotation=0)
-        plt.setp(fig.ax_heatmap.get_xticklabels(), rotation=90)
-
-        cmap_suffix = "" if cmap is None else f"_{cmap}"
-
-        # Save the clustermap to file
-        fig.savefig(f"blog/figures/clustermap_{plot_type}{cmap_suffix}.pdf")
-        fig.savefig(f"blog/figures/clustermap_{plot_type}{cmap_suffix}.png")
-        fig.savefig(f"blog/figures/clustermap_{plot_type}{cmap_suffix}_t.png", transparent=True, facecolor="none")

open-moe-llm-leaderboard-gh/cli/averitec-upload-cli.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-
-from datasets import load_dataset
-
-path = "pminervini/averitec"
-
-ds = load_dataset(
-    "json",
-    data_files={
-        "train": "/Users/pasquale/workspace/AVeriTeC/data/train.json",
-        "dev": "/Users/pasquale/workspace/AVeriTeC/data/dev.json",
-    },
-)
-ds.push_to_hub(path)

open-moe-llm-leaderboard-gh/cli/beta-cli.py

@@ -1,77 +0,0 @@
-#!/usr/bin/env python
-
-from huggingface_hub import snapshot_download
-from src.leaderboard.read_evals import get_raw_eval_results
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, RESULTS_REPO
-
-from src.backend.run_eval_suite import run_evaluation
-from src.backend.manage_requests import check_completed_evals, get_eval_requests, set_eval_request
-from src.backend.sort_queue import sort_models_by_priority
-from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Task
-
-from src.leaderboard.read_evals import get_raw_eval_results
-
-from src.backend.manage_requests import EvalRequest
-from src.leaderboard.read_evals import EvalResult
-
-snapshot_download(
-    repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-)
-snapshot_download(
-    repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-)
-
-PENDING_STATUS = "PENDING"
-RUNNING_STATUS = "RUNNING"
-FINISHED_STATUS = "FINISHED"
-FAILED_STATUS = "FAILED"
-
-TASKS_HARNESS = [task.value for task in Tasks]
-
-current_finished_status = [FINISHED_STATUS]
-
-
-def request_to_result_name(request: EvalRequest) -> str:
-    org_and_model = request.model.split("/", 1)
-    if len(org_and_model) == 1:
-        model = org_and_model[0]
-        res = f"{model}_{request.precision}"
-    else:
-        org = org_and_model[0]
-        model = org_and_model[1]
-        res = f"{org}_{model}_{request.precision}"
-    return res
-
-
-# Get all eval request that are FINISHED, if you want to run other evals, change this parameter
-eval_requests: list[EvalRequest] = get_eval_requests(
-    job_status=current_finished_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND
-)
-# Sort the evals by priority (first submitted first run)
-eval_requests: list[EvalRequest] = sort_models_by_priority(api=API, models=eval_requests)
-
-eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
-
-result_name_to_request = {request_to_result_name(r): r for r in eval_requests}
-result_name_to_result = {r.eval_name: r for r in eval_results}
-
-print("Requests", sorted(result_name_to_request.keys()))
-print("Results", sorted(result_name_to_result.keys()))
-
-for eval_request in eval_requests:
-    result_name: str = request_to_result_name(eval_request)
-
-    # Check the corresponding result
-    eval_result: EvalResult = result_name_to_result[result_name]
-
-    # Iterate over tasks and, if we do not have results for a task, run the relevant evaluations
-    for task in TASKS_HARNESS:
-        task_name = task.benchmark
-
-        if task_name not in eval_result.results:
-            print("RUN THIS ONE!", result_name, task_name)
-
-raw_data = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
-all_data_json = [v.to_dict() for v in raw_data if v.is_complete()]
-
-breakpoint()

open-moe-llm-leaderboard-gh/cli/completed-cli.py

@@ -1,136 +0,0 @@
-#!/usr/bin/env python
-
-from huggingface_hub import snapshot_download
-
-from src.backend.manage_requests import get_eval_requests
-from src.backend.sort_queue import sort_models_by_priority
-from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND
-
-from src.backend.manage_requests import EvalRequest
-from src.leaderboard.read_evals import EvalResult
-
-from src.envs import QUEUE_REPO, RESULTS_REPO, API
-
-import logging
-import pprint
-
-logging.getLogger("openai").setLevel(logging.WARNING)
-
-logging.basicConfig(level=logging.ERROR)
-pp = pprint.PrettyPrinter(width=80)
-
-PENDING_STATUS = "PENDING"
-RUNNING_STATUS = "RUNNING"
-FINISHED_STATUS = "FINISHED"
-FAILED_STATUS = "FAILED"
-
-TASKS_HARNESS = [task.value for task in Tasks]
-
-snapshot_download(
-    repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-)
-snapshot_download(
-    repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-)
-
-
-def request_to_result_name(request: EvalRequest) -> str:
-    org_and_model = request.model.split("/", 1)
-    if len(org_and_model) == 1:
-        model = org_and_model[0]
-        res = f"{model}_{request.precision}"
-    else:
-        org = org_and_model[0]
-        model = org_and_model[1]
-        res = f"{org}_{model}_{request.precision}"
-    return res
-
-
-def process_finished_requests() -> bool:
-    current_finished_status = [FINISHED_STATUS]
-
-    if False:
-        import os
-        import dateutil
-
-        model_result_filepaths = []
-        results_path = f"{EVAL_RESULTS_PATH_BACKEND}/EleutherAI/gpt-neo-1.3B"
-        requests_path = f"{EVAL_REQUESTS_PATH_BACKEND}/EleutherAI/gpt-neo-1.3B_eval_request_False_False_False.json"
-
-        for root, _, files in os.walk(results_path):
-            # We should only have json files in model results
-            if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-                continue
-
-            # Sort the files by date
-            try:
-                files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-            except dateutil.parser._parser.ParserError:
-                files = [files[-1]]
-
-            for file in files:
-                model_result_filepaths.append(os.path.join(root, file))
-
-        eval_results = {}
-        for model_result_filepath in model_result_filepaths:
-            # Creation of result
-            eval_result = EvalResult.init_from_json_file(model_result_filepath)
-            eval_result.update_with_request_file(requests_path)
-
-            print("XXX", eval_result)
-
-            # Store results of same eval together
-            eval_name = eval_result.eval_name
-            if eval_name in eval_results.keys():
-                eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-            else:
-                eval_results[eval_name] = eval_result
-
-        print(eval_results)
-
-        return True
-
-    # Get all eval request that are FINISHED, if you want to run other evals, change this parameter
-    eval_requests: list[EvalRequest] = get_eval_requests(
-        job_status=current_finished_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND
-    )
-    # Sort the evals by priority (first submitted first run)
-    eval_requests: list[EvalRequest] = sort_models_by_priority(api=API, models=eval_requests)
-
-    # XXX
-    # eval_requests = [r for r in eval_requests if 'neo-1.3B' in r.model]
-
-    import random
-
-    random.shuffle(eval_requests)
-
-    from src.leaderboard.read_evals import get_raw_eval_results
-
-    eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH_BACKEND, EVAL_REQUESTS_PATH_BACKEND)
-
-    result_name_to_request = {request_to_result_name(r): r for r in eval_requests}
-    result_name_to_result = {r.eval_name: r for r in eval_results}
-
-    for eval_request in eval_requests:
-        result_name: str = request_to_result_name(eval_request)
-
-        # Check the corresponding result
-        from typing import Optional
-
-        eval_result: Optional[EvalResult] = (
-            result_name_to_result[result_name] if result_name in result_name_to_result else None
-        )
-
-        # Iterate over tasks and, if we do not have results for a task, run the relevant evaluations
-        for task in TASKS_HARNESS:
-            task_name = task.benchmark
-
-            if eval_result is None or task_name not in eval_result.results:
-                eval_request: EvalRequest = result_name_to_request[result_name]
-
-                # print(eval_result)
-                print(result_name, "is incomplete -- missing task:", task_name, eval_result, eval_request.likes)
-
-
-if __name__ == "__main__":
-    res = process_finished_requests()

open-moe-llm-leaderboard-gh/cli/create_request_file.py

@@ -1,107 +0,0 @@
-import json
-import os
-import pprint
-import re
-from datetime import datetime, timezone
-
-import click
-from colorama import Fore
-from huggingface_hub import HfApi, snapshot_download
-
-EVAL_REQUESTS_PATH = "eval-queue"
-QUEUE_REPO = "sparse-generative-ai/requests"
-
-precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
-model_types = ("pretrained", "fine-tuned", "RL-tuned", "instruction-tuned")
-weight_types = ("Original", "Delta", "Adapter")
-
-
-def get_model_size(model_info, precision: str):
-    size_pattern = size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        try:
-            size_match = re.search(size_pattern, model_info.modelId.lower())
-            model_size = size_match.group(0)
-            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
-        except AttributeError:
-            return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-
-def main():
-    api = HfApi()
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-    snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH, repo_type="dataset")
-
-    model_name = click.prompt("Enter model name")
-    revision = click.prompt("Enter revision", default="main")
-    precision = click.prompt("Enter precision", default="float32", type=click.Choice(precisions))
-    model_type = click.prompt("Enter model type", type=click.Choice(model_types))
-    weight_type = click.prompt("Enter weight type", default="Original", type=click.Choice(weight_types))
-    base_model = click.prompt("Enter base model", default="")
-    status = click.prompt("Enter status", default="FINISHED")
-
-    try:
-        model_info = api.model_info(repo_id=model_name, revision=revision)
-    except Exception as e:
-        print(f"{Fore.RED}Could not find model info for {model_name} on the Hub\n{e}{Fore.RESET}")
-        return 1
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        license = "?"
-
-    eval_entry = {
-        "model": model_name,
-        "base_model": base_model,
-        "revision": revision,
-        "private": False,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": status,
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-    }
-
-    user_name = ""
-    model_path = model_name
-    if "/" in model_name:
-        user_name = model_name.split("/")[0]
-        model_path = model_name.split("/")[1]
-
-    pprint.pprint(eval_entry)
-
-    if click.confirm("Do you want to continue? This request file will be pushed to the hub"):
-        click.echo("continuing...")
-
-        out_dir = f"{EVAL_REQUESTS_PATH}/{user_name}"
-        os.makedirs(out_dir, exist_ok=True)
-        out_path = f"{out_dir}/{model_path}_eval_request_{False}_{precision}_{weight_type}.json"
-
-        with open(out_path, "w") as f:
-            f.write(json.dumps(eval_entry))
-
-        api.upload_file(
-            path_or_fileobj=out_path,
-            path_in_repo=out_path.split(f"{EVAL_REQUESTS_PATH}/")[1],
-            repo_id=QUEUE_REPO,
-            repo_type="dataset",
-            commit_message=f"Add {model_name} to eval queue",
-        )
-    else:
-        click.echo("aborting...")
-
-
-if __name__ == "__main__":
-    main()

open-moe-llm-leaderboard-gh/cli/eval-cli.py

@@ -1,96 +0,0 @@
-#!/usr/bin/env python
-
-from huggingface_hub import snapshot_download
-
-from src.backend.envs import EVAL_REQUESTS_PATH_BACKEND
-from src.backend.manage_requests import get_eval_requests
-from src.backend.manage_requests import EvalRequest
-from src.backend.run_eval_suite import run_evaluation
-
-from src.backend.tasks.xsum.task import XSum
-from src.backend.tasks.xsum.task_v2 import XSumv2
-
-from src.backend.tasks.cnndm.task import CNNDM
-from src.backend.tasks.cnndm.task_v2 import CNNDMv2
-
-from src.backend.tasks.selfcheckgpt.task import SelfCheckGPT
-
-from lm_eval.tasks import TaskManager
-from lm_eval import tasks, evaluator, utils
-
-from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND, DEVICE, LIMIT, Task
-from src.envs import QUEUE_REPO
-
-from lm_eval.models.huggingface import HFLM
-
-
-def main():
-    # snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-
-    PENDING_STATUS = "PENDING"
-    RUNNING_STATUS = "RUNNING"
-    FINISHED_STATUS = "FINISHED"
-    FAILED_STATUS = "FAILED"
-
-    status = [PENDING_STATUS, RUNNING_STATUS, FINISHED_STATUS, FAILED_STATUS]
-
-    # Get all eval request that are FINISHED, if you want to run other evals, change this parameter
-    eval_requests: list[EvalRequest] = get_eval_requests(
-        job_status=status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND, do_download=False
-    )
-    # eval_request = [r for r in eval_requests if 'bloom-560m' in r.model][0]
-    eval_request = [r for r in eval_requests if "meta-llama/Llama-2-7b-hf" in r.model][0]
-
-    # my_task = Task("memo-trap", "acc", "memo-trap", 0)
-    # my_task = Task("selfcheckgpt", "avg-selfcheckgpt", "SGPT", 2)
-    # my_task = Task("ifeval", "prompt_level_strict_acc", "IFEval", 0)
-    # my_task = Task("truefalse_cieacf", "acc", "TrueFalse", 5)
-    # my_task = Task("faithdial_hallu", "acc", "FaithDIAL", 2)
-
-    # my_task = Task("nq_swap", "exact_match", "NQ-Swap", 2)
-    # my_task = Task("memo-trap_v2", "acc", "XXX", 2)
-    my_task = Task("xsum_v2", "rougeL", "XXX", 0)
-    # my_task = Task("squadv2", "exact", "XXX", 0)
-    # my_task = Task("scrolls_qasper", "f1", "XXX", 0)
-
-    eval_logger = utils.eval_logger
-    import logging
-
-    eval_logger.setLevel(getattr(logging, "DEBUG"))
-
-    TASKS_HARNESS = [my_task]
-    # task_names = ['triviaqa']
-    # TASKS_HARNESS = [task.value for task in Tasks]
-
-    # include_task_folder("src/backend/tasks/")
-    task_manager = TaskManager(include_path="./src/backend/tasks/")
-    # task_manager.initialize_tasks(include_path="src/backend/tasks/")
-
-    # breakpoint()
-
-    print(task_manager.all_tasks)
-
-    for task in TASKS_HARNESS:
-        print(f"Selected Tasks: [{task}]")
-        import torch
-
-        # breakpoint()
-        results = evaluator.simple_evaluate(
-            model="hf",
-            model_args=eval_request.get_model_args(),
-            tasks=[task.benchmark],
-            num_fewshot=task.num_fewshot,
-            batch_size=1,
-            device="mps",
-            use_cache=None,
-            limit=2,
-            write_out=True,
-            task_manager=task_manager,
-        )
-        print("AAA", results["results"])
-
-        breakpoint()
-
-
-if __name__ == "__main__":
-    main()

open-moe-llm-leaderboard-gh/cli/fever-upload-cli.py

@@ -1,70 +0,0 @@
-#!/usr/bin/env python3
-
-import glob
-import os
-
-import random
-from tqdm import tqdm
-
-from datasets import Dataset, DatasetDict, load_dataset
-
-
-def convert(list_of_dicts):
-    res = {}
-    for d in list_of_dicts:
-        for k, v in d.items():
-            res.setdefault(k, []).append(v)
-    return res
-
-
-v10 = load_dataset("fever", "v1.0")
-name_lst = ["train", "labelled_dev"]
-
-old_to_new_label_map = {"SUPPORTS": "supported", "REFUTES": "refuted"}
-
-data_map = {}
-
-for name in name_lst:
-    instance_lst = []
-
-    for entry in tqdm(v10[name]):
-        id_ = entry["id"]
-        label = entry["label"]
-        claim = entry["claim"]
-
-        evidence_id = entry["evidence_id"]
-        evidence_wiki_url = entry["evidence_wiki_url"]
-
-        if evidence_id != -1:
-            assert label in {"SUPPORTS", "REFUTES"}
-
-            instance = {"id": id_, "label": old_to_new_label_map[label], "claim": claim}
-            instance_lst.append(instance)
-
-    key = "dev" if name in {"labelled_dev"} else name
-
-    instance_lst = sorted([dict(t) for t in {tuple(d.items()) for d in instance_lst}], key=lambda d: d["claim"])
-
-    label_to_instance_lst = {}
-    for e in instance_lst:
-        if e["label"] not in label_to_instance_lst:
-            label_to_instance_lst[e["label"]] = []
-        label_to_instance_lst[e["label"]].append(e)
-
-    min_len = min(len(v) for k, v in label_to_instance_lst.items())
-
-    new_instance_lst = []
-    for k in sorted(label_to_instance_lst.keys()):
-        new_instance_lst += label_to_instance_lst[k][:min_len]
-
-    random.Random(42).shuffle(new_instance_lst)
-    data_map[key] = new_instance_lst
-
-ds_path = "pminervini/hl-fever"
-
-task_to_ds_map = {k: Dataset.from_dict(convert(v)) for k, v in data_map.items()}
-ds_dict = DatasetDict(task_to_ds_map)
-
-ds_dict.push_to_hub(ds_path, "v1.0")
-
-# breakpoint()

open-moe-llm-leaderboard-gh/cli/fix-requests-cli.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python
-
-import os
-import fnmatch
-
-import json
-from huggingface_hub import HfApi
-
-
-def find_json_files(directory):
-    matches = []
-    for root, dirnames, filenames in os.walk(directory):
-        for filename in fnmatch.filter(filenames, "*.json"):
-            matches.append(os.path.join(root, filename))
-    return matches
-
-
-directory_path = "/Users/pasquale/workspace/eval/requests"
-json_files = find_json_files(directory_path)
-
-api = HfApi()
-model_lst = api.list_models()
-
-model_lst = [m for m in model_lst]
-
-id_to_model = {m.id: m for m in model_lst}
-
-for path in json_files:
-    with open(path, "r") as fr:
-        data = json.load(fr)
-
-        model_id = data["model"]
-        if model_id in id_to_model:
-            model = id_to_model[model_id]
-
-            to_overwrite = False
-
-            is_finetuned = any(tag.startswith("base_model:") for tag in id_to_model[data["model"]].tags)
-
-            if is_finetuned:
-                data["model_type"] = "fine-tuned"
-                to_overwrite = True
-
-            is_instruction_tuned = ("nstruct" in model_id) or ("chat" in model_id)
-            if is_instruction_tuned:
-                data["model_type"] = "instruction-tuned"
-                to_overwrite = True
-
-            if to_overwrite is True:
-                with open(path, "w") as fw:
-                    json.dump(data, fw)
-
-        else:
-            print(f"Model {model_id} not found")

open-moe-llm-leaderboard-gh/cli/halueval-upload-cli.py

@@ -1,72 +0,0 @@
-#!/usr/bin/env python3
-
-import random
-import requests
-
-from datasets import load_dataset, Dataset, DatasetDict
-
-
-path = "pminervini/HaluEval"
-
-API_URL = f"https://datasets-server.huggingface.co/splits?dataset={path}"
-response = requests.get(API_URL)
-res_json = response.json()
-
-gold_splits = {"dialogue", "qa", "summarization", "general"}
-
-available_splits = {split["config"] for split in res_json["splits"]} if "splits" in res_json else set()
-
-name_to_ds = dict()
-
-for name in gold_splits:
-    ds = load_dataset("json", data_files={"data": f"data/{name}_data.json"})
-    name_to_ds[name] = ds
-    # if name not in available_splits:
-    ds.push_to_hub(path, config_name=name)
-
-
-def list_to_dict(lst: list) -> dict:
-    res = dict()
-    for entry in lst:
-        for k, v in entry.items():
-            if k not in res:
-                res[k] = []
-            res[k] += [v]
-    return res
-
-
-for name in gold_splits - {"general"}:
-    random.seed(42)
-    ds = name_to_ds[name]
-    new_entry_lst = []
-
-    for entry in ds["data"]:
-        is_hallucinated = random.random() > 0.5
-        new_entry = None
-        if name in {"qa"}:
-            new_entry = {
-                "knowledge": entry["knowledge"],
-                "question": entry["question"],
-                "answer": entry[f'{"hallucinated" if is_hallucinated else "right"}_answer'],
-                "hallucination": "yes" if is_hallucinated else "no",
-            }
-        if name in {"dialogue"}:
-            new_entry = {
-                "knowledge": entry["knowledge"],
-                "dialogue_history": entry["dialogue_history"],
-                "response": entry[f'{"hallucinated" if is_hallucinated else "right"}_response'],
-                "hallucination": "yes" if is_hallucinated else "no",
-            }
-        if name in {"summarization"}:
-            new_entry = {
-                "document": entry["document"],
-                "summary": entry[f'{"hallucinated" if is_hallucinated else "right"}_summary'],
-                "hallucination": "yes" if is_hallucinated else "no",
-            }
-        assert new_entry is not None
-        new_entry_lst += [new_entry]
-    new_ds_map = list_to_dict(new_entry_lst)
-    new_ds = Dataset.from_dict(new_ds_map)
-    new_dsd = DatasetDict({"data": new_ds})
-
-    new_dsd.push_to_hub(path, config_name=f"{name}_samples")

open-moe-llm-leaderboard-gh/cli/isp-upload-cli.py

@@ -1,20 +0,0 @@
-#!/usr/bin/env python3
-
-import glob
-import os
-
-from datasets import load_dataset
-
-folder_path = "isp-data-json/"  # Replace with your folder path
-
-# Search for all .json files in the folder
-json_files = glob.glob(os.path.join(folder_path, "*.jsonl"))
-
-path = "pminervini/inverse-scaling"
-
-for json_path in json_files:
-    base_name = os.path.basename(json_path)
-    name = base_name.split("_")[0]
-
-    ds = load_dataset("json", data_files={"data": json_path})
-    ds.push_to_hub(path, config_name=name)

open-moe-llm-leaderboard-gh/cli/nqswap-upload-cli.py

@@ -1,8 +0,0 @@
-#!/usr/bin/env python3
-
-from datasets import load_dataset
-
-path = "pminervini/NQ-Swap"
-
-ds = load_dataset("json", data_files={"original": "nqswap/original.jsonl", "substituted": "nqswap/substituted.jsonl"})
-ds.push_to_hub(path)

open-moe-llm-leaderboard-gh/cli/shroom-upload-cli.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-from datasets import Dataset, DatasetDict
-
-file_path = "shroom-data/val.model-agnostic.json"
-ds_path = "pminervini/shroom"
-
-with open(file_path, "r") as file:
-    data = json.load(file)
-
-
-def convert(list_of_dicts):
-    dict_of_lists = {}
-    for d in list_of_dicts:
-        for key, value in d.items():
-            dict_of_lists.setdefault(key, []).append(value)
-    return dict_of_lists
-
-
-task_to_data_map = {}
-
-for entry in data:
-    task_name = entry["task"]
-    del entry["task"]
-    if task_name not in task_to_data_map:
-        task_to_data_map[task_name] = []
-    task_to_data_map[task_name] += [entry]
-
-task_to_ds_map = {k: Dataset.from_dict(convert(data)) for k, data in task_to_data_map.items()}
-
-ds_dict = DatasetDict(task_to_ds_map)
-
-ds_dict.push_to_hub(ds_path)

open-moe-llm-leaderboard-gh/cli/submit-cli.py

@@ -1,194 +0,0 @@
-#!/usr/bin/env python
-
-import json
-import os
-import time
-
-from datetime import datetime, timezone
-
-from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO
-from src.submission.check_validity import already_submitted_models, get_model_size, is_model_on_hub
-
-from huggingface_hub import snapshot_download
-from src.backend.envs import EVAL_REQUESTS_PATH_BACKEND
-from src.backend.manage_requests import get_eval_requests
-from src.backend.manage_requests import EvalRequest
-
-
-def add_new_eval(
-    model: str, base_model: str, revision: str, precision: str, private: bool, weight_type: str, model_type: str
-):
-    REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        tokens = model.split("/")
-        user_name = tokens[0]
-        model_path = tokens[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return print("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(
-            model_name=base_model, revision=revision, token=H4_TOKEN, test_tokenizer=True
-        )
-        if not base_model_on_hub:
-            print(f'Base model "{base_model}" {error}')
-            return
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, test_tokenizer=True)
-        if not model_on_hub:
-            print(f'Model "{model}" {error}')
-            return
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        print("Could not get your model information. Please fill it up properly.")
-        return
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    license = "none"
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        print("Please select a license for your model")
-        # return
-
-    # modelcard_OK, error_msg = check_model_card(model)
-    # if not modelcard_OK:
-    #     print(error_msg)
-    #     return
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "private": private,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        print("This model has been already submitted.")
-        return
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    print(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )
-    return
-
-
-def main():
-    from huggingface_hub import HfApi
-
-    api = HfApi()
-    model_lst = api.list_models()
-
-    model_lst = [m for m in model_lst]
-
-    def custom_filter(m) -> bool:
-        # res = m.pipeline_tag in {'text-generation'} and 'en' in m.tags and m.private is False
-        # res = m.pipeline_tag in {'text-generation'} and 'en' in m.tags and m.private is False and 'mistralai/' in m.id
-        res = "mistralai/" in m.id
-        return res
-
-    filtered_model_lst = sorted([m for m in model_lst if custom_filter(m)], key=lambda m: m.downloads, reverse=True)
-
-    snapshot_download(
-        repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60
-    )
-
-    PENDING_STATUS = "PENDING"
-    RUNNING_STATUS = "RUNNING"
-    FINISHED_STATUS = "FINISHED"
-    FAILED_STATUS = "FAILED"
-
-    status = [PENDING_STATUS, RUNNING_STATUS, FINISHED_STATUS, FAILED_STATUS]
-
-    # Get all eval requests
-    eval_requests: list[EvalRequest] = get_eval_requests(
-        job_status=status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND
-    )
-
-    requested_model_names = {e.model for e in eval_requests}
-
-    # breakpoint()
-
-    for i in range(min(200, len(filtered_model_lst))):
-        model = filtered_model_lst[i]
-
-        print(f"Considering {model.id} ..")
-
-        is_finetuned = any(tag.startswith("base_model:") for tag in model.tags)
-
-        model_type = "pretrained"
-        if is_finetuned:
-            model_type = "fine-tuned"
-
-        is_instruction_tuned = "nstruct" in model.id
-        if is_instruction_tuned:
-            model_type = "instruction-tuned"
-
-        if model.id not in requested_model_names:
-
-            if "mage" not in model.id:
-                add_new_eval(
-                    model=model.id,
-                    base_model="",
-                    revision="main",
-                    precision="float32",
-                    private=False,
-                    weight_type="Original",
-                    model_type=model_type,
-                )
-                time.sleep(10)
-        else:
-            print(f"Model {model.id} already added, not adding it to the queue again.")
-
-
-if __name__ == "__main__":
-    main()

open-moe-llm-leaderboard-gh/cli/sync-open-llm-cli.py

@@ -1,109 +0,0 @@
-import os
-import json
-import glob
-
-from tqdm import tqdm
-from huggingface_hub import HfApi, snapshot_download
-from src.backend.manage_requests import EvalRequest
-from src.backend.envs import EVAL_REQUESTS_PATH_BACKEND_SYNC
-from src.envs import QUEUE_REPO, API
-from src.envs import EVAL_REQUESTS_PATH_OPEN_LLM, QUEUE_REPO_OPEN_LLM
-from src.utils import my_snapshot_download
-
-
-def my_set_eval_request(api, json_filepath, hf_repo, local_dir):
-    for i in range(10):
-        try:
-            set_eval_request(api=api, json_filepath=json_filepath, hf_repo=hf_repo, local_dir=local_dir)
-            return
-        except Exception:
-            time.sleep(60)
-    return
-
-
-def set_eval_request(api: HfApi, json_filepath: str, hf_repo: str, local_dir: str):
-    """Updates a given eval request with its new status on the hub (running, completed, failed, ...)"""
-
-    with open(json_filepath) as fp:
-        data = json.load(fp)
-
-    with open(json_filepath, "w") as f:
-        f.write(json.dumps(data))
-
-    api.upload_file(
-        path_or_fileobj=json_filepath,
-        path_in_repo=json_filepath.replace(local_dir, ""),
-        repo_id=hf_repo,
-        repo_type="dataset",
-    )
-
-
-def get_request_file_for_model(data, requests_path):
-    model_name = data["model"]
-    precision = data["precision"]
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED and RUNNING"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if req_content["precision"] == precision.split(".")[-1]:
-                request_file = tmp_request_file
-    return request_file
-
-
-def update_model_type(data, requests_path):
-    open_llm_request_file = get_request_file_for_model(data, requests_path)
-
-    try:
-        with open(open_llm_request_file, "r") as f:
-            open_llm_request = json.load(f)
-            data["model_type"] = open_llm_request["model_type"]
-            return True, data
-    except:
-        return False, data
-
-
-def read_and_write_json_files(directory, requests_path_open_llm):
-    # Walk through the directory
-    for subdir, dirs, files in tqdm(os.walk(directory), desc="updating model type according to open llm leaderboard"):
-        for file in files:
-            # Check if the file is a JSON file
-            if file.endswith(".json"):
-                file_path = os.path.join(subdir, file)
-                # Open and read the JSON file
-                with open(file_path, "r") as json_file:
-                    data = json.load(json_file)
-                sucess, data = update_model_type(data, requests_path_open_llm)
-                if sucess:
-                    with open(file_path, "w") as json_file:
-                        json.dump(data, json_file)
-                    my_set_eval_request(
-                        api=API, json_filepath=file_path, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND_SYNC
-                    )
-
-
-if __name__ == "__main__":
-    my_snapshot_download(
-        repo_id=QUEUE_REPO_OPEN_LLM,
-        revision="main",
-        local_dir=EVAL_REQUESTS_PATH_OPEN_LLM,
-        repo_type="dataset",
-        max_workers=60,
-    )
-    my_snapshot_download(
-        repo_id=QUEUE_REPO,
-        revision="main",
-        local_dir=EVAL_REQUESTS_PATH_BACKEND_SYNC,
-        repo_type="dataset",
-        max_workers=60,
-    )
-    read_and_write_json_files(EVAL_REQUESTS_PATH_BACKEND_SYNC, EVAL_REQUESTS_PATH_OPEN_LLM)

open-moe-llm-leaderboard-gh/cli/truefalse-upload-cli.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python3
-
-import glob
-import os
-
-from datasets import load_dataset
-
-path = "pminervini/true-false"
-folder_path = "true-false-data/"  # Replace with your folder path
-
-# Search for all .json files in the folder
-csv_files = glob.glob(os.path.join(folder_path, "*.csv"))
-
-ds = load_dataset("csv", data_files={os.path.basename(path).split("_")[0]: path for path in csv_files})
-ds.push_to_hub(path)

open-moe-llm-leaderboard-gh/pyproject.toml

@@ -1,13 +0,0 @@
-[tool.ruff]
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
-line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
-
-[tool.isort]
-profile = "black"
-line_length = 119
-
-[tool.black]
-line-length = 119

open-moe-llm-leaderboard-gh/requirements.txt

@@ -1,34 +0,0 @@
-torch
-colorama
-APScheduler
-black
-click
-datasets
-gradio
-gradio_client
-huggingface-hub
-matplotlib
-numpy
-pandas
-plotly
-python-dateutil
-requests
-semantic-version
-tqdm
-wandb
-transformers>=4.36.0
-tokenizers>=0.15.0
-lm_eval[ifeval] @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@v0.4.2
-accelerate
-sentencepiece
-langdetect
-sacrebleu
-cchardet
-rouge_score
-bert-score
-evaluate
-spacy==3.7.4
-selfcheckgpt
-immutabledict
-gputil
-bitsandbytes

open-moe-llm-leaderboard-gh/snippets/xsum.yaml

@@ -1,49 +0,0 @@
-task: xsum
-dataset_path: EdinburghNLP/xsum
-dataset_name: xsum
-output_type: generate_until
-training_split: train
-validation_split: validation
-test_split: test
-doc_to_text: "Document: {{document}}\nSummary:"
-doc_to_target: "{{summary}}"
-# process_docs: !function utils.process_docs
-process_results: !function utils.process_results
-should_decontaminate: True
-doc_to_decontamination_query: document
-generation_kwargs:
-  until:
-    - "\n"
-    - "."
-  do_sample: false
-  temperature: 0.0
-metric_list:
-  - metric: rouge1_max
-    aggregation: mean
-    higher_is_better: true
-  - metric: rouge1_acc
-    aggregation: mean
-    higher_is_better: true
-  - metric: rouge1_diff
-    aggregation: mean
-    higher_is_better: true
-  - metric: rouge2_max
-    aggregation: mean
-    higher_is_better: true
-  - metric: rouge2_acc
-    aggregation: mean
-    higher_is_better: true
-  - metric: rouge2_diff
-    aggregation: mean
-    higher_is_better: true
-  - metric: rougeL_max
-    aggregation: mean
-    higher_is_better: true
-  - metric: rougeL_acc
-    aggregation: mean
-    higher_is_better: true
-  - metric: rougeL_diff
-    aggregation: mean
-    higher_is_better: true
-metadata:
-  - version: 0.0

open-moe-llm-leaderboard-gh/src/backend/envs.py

@@ -1,68 +0,0 @@
-import os
-
-import torch
-
-from dataclasses import dataclass
-from enum import Enum
-
-from src.envs import CACHE_PATH
-
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-    num_fewshot: int
-
-
-class Tasks(Enum):
-    # task0 = Task("nq_open", "em", "NQ Open", 64)  # 64, as in the ATLAS paper
-    # task1 = Task("triviaqa", "em", "TriviaQA", 64)  # 64, as in the ATLAS paper
-
-    # task11 = Task("nq8", "em", "NQ Open 8", 8)
-    # task12 = Task("tqa8", "em", "TriviaQA 8", 8)
-
-    # TruthfulQA is intended as a zero-shot benchmark [5, 47]. https://owainevans.github.io/pdfs/truthfulQA_lin_evans.pdf
-    # task2 = Task("truthfulqa_gen", "rougeL_acc", "TruthfulQA Gen", 0)
-    # task3 = Task("truthfulqa_mc1", "acc", "TruthfulQA MC1", 0)
-    # task4 = Task("truthfulqa_mc2", "acc", "TruthfulQA MC2", 0)
-
-    # task5 = Task("halueval_qa", "acc", "HaluEval QA", 0)
-    # task6 = Task("halueval_dialogue", "acc", "HaluEval Dialogue", 0)
-    # task7 = Task("halueval_summarization", "acc", "HaluEval Summarization", 0)
-
-    # task8 = Task("xsum", "rougeL", "XSum", 2)
-    # task9 = Task("cnndm", "rougeL", "CNN/DM", 2)
-
-    # task8_1 = Task("xsum_v2", "rougeL", "XSum", 0)
-    # task9_1 = Task("cnndm_v2", "rougeL", "CNN/DM", 0)
-
-    # task10 = Task("memo-trap", "acc", "memo-trap", 0)
-    # task10_2 = Task("memo-trap_v2", "acc", "memo-trap", 0)
-
-    # task13 = Task("ifeval", "prompt_level_strict_acc", "IFEval", 0)
-
-    task14 = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT", 0)
-
-    # task15 = Task("fever10", "acc", "FEVER", 16)
-    # task15_1 = Task("fever11", "acc", "FEVER", 8)
-
-    # task16 = Task("squadv2", "exact", "SQuADv2", 4)
-
-    # task17 = Task("truefalse_cieacf", "acc", "TrueFalse", 8)
-
-    # task18 = Task("faithdial_hallu", "acc", "FaithDial", 8)
-    # task19 = Task("faithdial_hallu_v2", "acc", "FaithDial", 8)
-
-    # task20 = Task("race", "acc", "RACE", 0)
-    task21 = Task("mmlu", "acc", "MMLU", 5)
-    task22 = Task("gsm8k_custom", "em", "GSM8K", 5)
-    task23 = Task("gsm8k_cot", "em", "GSM8K", 8)
-
-
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_REQUESTS_PATH_BACKEND_SYNC = os.path.join(CACHE_PATH, "eval-queue-bk-sync")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu"

open-moe-llm-leaderboard-gh/src/backend/hflm_with_measurement.py

@@ -1,592 +0,0 @@
-import copy
-import os
-from datetime import timedelta
-import sys
-from time import time
-from pathlib import Path
-from typing import List, Literal, Optional, Tuple, Union
-
-import torch
-import torch.nn.functional as F
-import transformers
-from accelerate import (
-    Accelerator,
-    DistributedType,
-    InitProcessGroupKwargs,
-    find_executable_batch_size,
-)
-from packaging import version
-from peft import PeftModel
-from peft import __version__ as PEFT_VERSION
-from tqdm import tqdm
-from transformers.models.auto.modeling_auto import (
-    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
-    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
-)
-from transformers import TextStreamer
-
-from lm_eval import utils
-from lm_eval.api.instance import Instance
-from lm_eval.api.model import TemplateLM
-from lm_eval.api.registry import register_model
-from lm_eval.models.utils import (
-    Collator,
-    clear_torch_cache,
-    get_dtype,
-    pad_and_concat,
-    stop_sequences_criteria,
-)
-from lm_eval.models.huggingface import HFLM
-from src.utils import get_gpu_number, get_gpu_details, get_peak_bw, transfer_precision2bytes, get_peak_flops
-from src.submission.check_validity import get_model_size
-from src.envs import API
-
-
-class StopWatch(TextStreamer):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.start_prefilling = None
-        self.prefilling_time = None
-        self.start_decoding = None
-        self.decoding_time = None
-        self.decoding_iterations = 0
-
-    def put(self, value):
-        if self.start_prefilling is None:
-            self.start_prefilling = time()
-            return
-        elif self.prefilling_time is None:
-            self.prefilling_time = time() - self.start_prefilling
-            self.start_decoding = time()
-        self.decoding_iterations += 1
-        return
-
-    def end(self):
-        if self.decoding_time is None and self.start_decoding is not None:
-            self.decoding_time = time() - self.start_decoding
-        return
-
-
-class HFLMWithMeasurement(HFLM):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.pretrained = kwargs.get("pretrained", None)
-        self.revision = kwargs.get("revision", None)
-        self.precision = kwargs.get("dtype", None)
-
-    def _loglikelihood_tokens(
-        self,
-        requests: List[Tuple[Tuple[str, str], List[int], List[int]]],
-        disable_tqdm: bool = False,
-        override_bs: int = None,
-    ) -> List[Tuple[float, bool]]:
-        # TODO: implement some kind of efficient-request-middleware that lumps together requests with the same context
-        res = []
-
-        def _collate(req: Tuple[Tuple[str, str], List[int], List[int]]):
-            """Defines the key for the sorted method"""
-            # the negative sign on len(toks) sorts descending - this has a few advantages:
-            # - time estimates will always be over not underestimates, which is more useful for planning
-            # - to know the size of a batch when going through the list, you know the first one is always the batch
-            #   padded context length. this is useful to simplify the batching logic and more importantly to make
-            #   automatic adaptive batches much much easier to implement
-            # - any OOMs will happen right away rather than near the end
-
-            toks = req[1] + req[2]
-            return -len(toks), tuple(toks)
-
-        def _lookup_one_token_cont(req: Tuple[Tuple[str, str], List[int], List[int]]):
-            """Defines the key to group and lookup one-token continuations"""
-            # Use with group_by="contexts" (optional)"
-            # allows for the creation of a lookup, so we can reuse logits in case of one-token continuations.
-            # speeds up some multiple-choice tasks proportionally to the number of choices.
-            # groups requests by context+continuation[:-1] and infer on one request/group.
-            return req[-2] + req[-1][:-1]
-
-        re_ord = Collator(
-            requests,
-            sort_fn=_collate,
-            group_by="contexts"
-            if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM
-            and self.logits_cache
-            else None,
-            group_fn=_lookup_one_token_cont,
-        )
-
-        # automatic (variable) batch size detection for vectorization
-        # pull longest context sample from request
-        n_reordered_requests = len(re_ord)
-        batch_size = (
-            self.batch_size
-            if self.batch_size != "auto"
-            else override_bs
-            if override_bs is not None
-            else 0
-        )
-        batch_fn = (
-            self._batch_scheduler
-            if self.batch_size == "auto"
-            and n_reordered_requests > 0
-            and not override_bs
-            else None
-        )
-
-        chunks = re_ord.get_batched(n=batch_size, batch_fn=batch_fn)
-        pbar = tqdm(
-            total=len(requests),
-            disable=(disable_tqdm or (self.rank != 0)),
-            desc="Running loglikelihood requests",
-        )
-        for chunk in chunks:
-            inps = []
-            cont_toks_list = []
-            inplens = []
-
-            conts = []
-            encoder_attns = []
-
-            padding_len_inp = None
-            padding_len_cont = None
-            # because vectorizing is annoying, we first convert each (context, continuation) pair to padded
-            # tensors, then we pack them together into a batch, call the model, and then pick it all apart
-            # again because vectorizing is annoying
-
-            for _, context_enc, continuation_enc in chunk:
-                # sanity check
-                assert len(context_enc) > 0
-                assert len(continuation_enc) > 0
-                assert len(continuation_enc) <= self.max_length
-
-                # how this all works (illustrated on a causal decoder-only setup):
-                #          CTX      CONT
-                # inp    0 1 2 3|4 5 6 7 8 9   <- last token is deleted by inp[:, :-1]
-                # model  \               \
-                # logits   1 2 3|4 5 6 7 8 9   <- the ctx half gets tossed out by the
-                # cont_toks      4 5 6 7 8 9      [:, -len(continuation_enc):, :self.vocab_size] slice
-
-                # when too long to fit in context, truncate from the left
-                if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM:
-                    inp = torch.tensor(
-                        (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1],
-                        dtype=torch.long,
-                        device=self.device,
-                    )
-                    (inplen,) = inp.shape
-                elif self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM:
-                    inp = torch.tensor(
-                        (context_enc)[-self.max_length :],
-                        dtype=torch.long,
-                        device=self.device,
-                    )
-                    (inplen,) = inp.shape
-
-                    # build encoder attn masks
-                    encoder_attns.append(torch.ones_like(inp))
-
-                    cont = torch.tensor(
-                        (continuation_enc)[-self.max_length :],
-                        # TODO: left-shift these?
-                        # TODO: our code assumes we never end up truncating conts for either model type
-                        dtype=torch.long,
-                        device=self.device,
-                    )
-                    (contlen,) = cont.shape
-
-                    conts.append(cont)
-
-                    padding_len_cont = (
-                        max(padding_len_cont, contlen)
-                        if padding_len_cont is not None
-                        else contlen
-                    )
-
-                padding_len_inp = (
-                    max(padding_len_inp, inplen)
-                    if padding_len_inp is not None
-                    else inplen
-                )
-
-                inps.append(inp)  # [1, inp_length]
-                cont_toks_list.append(continuation_enc)
-                inplens.append(inplen)
-
-            # create encoder attn mask and batched conts, if seq2seq
-            call_kwargs = {}
-            if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM:
-                batched_inps = pad_and_concat(
-                    padding_len_inp, inps, padding_side="right"
-                )  # [batch, padding_len_inp]
-            elif self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM:
-                # TODO: left-pad encoder inps and mask?
-                batched_inps = pad_and_concat(
-                    padding_len_inp, inps
-                )  # [batch, padding_len_inp]
-                batched_conts = pad_and_concat(
-                    padding_len_cont, conts
-                )  # [batch, padding_len_cont]
-                batched_encoder_mask = pad_and_concat(
-                    padding_len_inp, encoder_attns
-                )  # [batch, padding_len_inp]
-                call_kwargs = {
-                    "attn_mask": batched_encoder_mask,
-                    "labels": batched_conts,
-                }
-
-            start = time()
-            intermediate_res = self._model_call(batched_inps, **call_kwargs)
-            end = time()
-            multi_logits = F.log_softmax(
-                intermediate_res , dim=-1
-            )  # [batch, padding_length (inp or cont), vocab]
-            per_sample_time = (end - start) / len(multi_logits)
-
-            for (request_str, ctx_tokens, _), logits, inplen, cont_toks in zip(
-                chunk, multi_logits, inplens, cont_toks_list
-            ):
-                # Slice to original seq length
-                contlen = len(cont_toks)
-                # take only logits in the continuation
-                # (discard context toks if decoder-only ; discard right-padding)
-                # also discards + checks for "virtual tokens" in the causal LM's input window
-                # from prompt/prefix tuning tokens, if applicable
-                ctx_len = (
-                    inplen + (logits.shape[0] - padding_len_inp)
-                    if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM
-                    else None
-                )
-                logits = self._select_cont_toks(logits, contlen=contlen, inplen=ctx_len)
-                logits = logits.unsqueeze(0)  # [1, seq, vocab]
-
-                # Check if per-token argmax is exactly equal to continuation
-                greedy_tokens = logits.argmax(dim=-1)
-
-                # check for one-token continuation cache hits.
-                # noop in case group_by != "contexts" or no cache hit and returns the
-                # original args. Otherwise, expands the logits batch dimension and yields each
-                # batch along with matching continuation tokens and prompt strings.
-                # logits -> [1, seq, vocab]
-                for request_str, cont_toks, logits in re_ord.get_cache(
-                    req_str=request_str,
-                    cxt_toks=ctx_tokens,
-                    cont_toks=cont_toks,
-                    logits=logits,
-                ):
-                    cont_toks = torch.tensor(
-                        cont_toks, dtype=torch.long, device=self.device
-                    ).unsqueeze(0)  # [1, seq]
-                    max_equal = (greedy_tokens == cont_toks).all()
-
-                    # Obtain log-probs at the corresponding continuation token indices
-                    # last_token_slice = logits[:, -1, :].squeeze(0).tolist()
-                    logits = torch.gather(logits, 2, cont_toks.unsqueeze(-1)).squeeze(
-                        -1
-                    )  # [1, seq]
-
-                    # Answer: (log prob, is-exact-match)
-                    answer = (float(logits.sum()), bool(max_equal))
-
-                    res.append((answer, per_sample_time, 0, 0, 0, 0))
-
-                    self.cache_hook.add_partial("loglikelihood", request_str, answer)
-                    pbar.update(1)
-
-        pbar.close()
-
-        return re_ord.get_original(res)
-
-    def _model_generate(self, context, max_tokens, stop, **generation_kwargs):
-        # temperature = 0.0 if not set
-        # if do_sample is false and temp==0.0:
-        # remove temperature, as do_sample=False takes care of this
-        # and we don't want a warning from HF
-        generation_kwargs["temperature"] = generation_kwargs.get("temperature", 0.0)
-        do_sample = generation_kwargs.get("do_sample", None)
-        
-        # is_gsm8k = generation_kwargs.get("is_gsm8k", False)
-
-        # The temperature has to be a strictly positive float -- if it is 0.0, use greedy decoding strategies
-        if generation_kwargs.get("temperature") == 0.0 and do_sample is None:
-            generation_kwargs["do_sample"] = do_sample = False
-
-        if do_sample is False and generation_kwargs.get("temperature") == 0.0:
-            generation_kwargs.pop("temperature")
-        
-        # if is_gsm8k:
-        #     generation_kwargs.pop("is_gsm8k")
-            
-        context_length = context.shape[1]
-
-        if self.model.__class__.__name__ == "MoE":
-            model_config = self.model.model.config
-        else:
-            model_config = self.model.config
-        
-        if not self.precision:
-            if model_config.quantization_config._load_in_4bit:
-                self.precision = "4bit"
-            elif model_config.quantization_config._load_in_8bit:
-                self.precision = "8bit"
-            else:
-                raise ValueError("Unknown precision")
-            
-        # print(self.model)
-        linear_count = 0 
-        element_wise_mul = 0
-        for name, module in self.model.named_modules():
-            if ('layers.0.' in name or 'decoder.0.' in name) and ('attn' not in name):
-                if 'experts.0.' in name:
-                    if isinstance(module, torch.nn.Linear):
-                        # print(name, module)
-                        linear_count += 1
-                elif 'experts' not in name:
-                    if "gate" not in name or "gate_proj" in name:
-                        if "gate_proj" in name:
-                            element_wise_mul = 1
-                        if isinstance(module, torch.nn.Linear):
-                            # print(name, module)
-                            linear_count += 1
-                else:
-                    continue
-        print(f"linear_count: {linear_count}")
-
-        stopping_criteria = stop_sequences_criteria(
-            self.tokenizer, stop, context.shape[1], context.shape[0]
-        )
-        stop_watch = StopWatch(self.tokenizer)
-        start = time()
-        res = self.model.generate(
-            input_ids=context,
-            max_new_tokens=max_tokens,
-            stopping_criteria=stopping_criteria,
-            pad_token_id=self.tokenizer.pad_token_id,
-            use_cache=True,
-            streamer=stop_watch,
-            **generation_kwargs,
-        )
-        end = time()
-
-        batch_size = context.shape[0]
-        output_length = stop_watch.decoding_iterations
-        
-        precision_bytes = transfer_precision2bytes(self.precision)
-        
-        model_info = API.model_info(repo_id=self.pretrained, revision=self.revision)
-        model_size_param = get_model_size(model_info=model_info, precision=self.precision)
-
-        n_layers = model_config.num_hidden_layers if hasattr(model_config, "num_hidden_layers") else model_config.num_layers
-        d_model = model_config.hidden_size if hasattr(model_config, "hidden_size") else model_config.d_model
-        
-        if hasattr(model_config, "num_experts_per_tok"):
-            n_experts_per_tok = model_config.num_experts_per_tok
-        elif hasattr(model_config, "num_selected_experts"):
-            n_experts_per_tok = model_config.num_selected_experts
-        else:
-            n_experts_per_tok = 1
-        
-        if hasattr(model_config, "ffn_dim"):
-            d_ff = model_config.ffn_dim
-        elif hasattr(model_config, "intermediate_size"):
-            d_ff = model_config.intermediate_size
-        elif hasattr(model_config, "d_ff"):
-            d_ff = model_config.d_ff
-        else:
-            if hasattr(model_config, "ff_ratio"):
-                d_ff = d_model * model_config.ff_ratio
-            else:
-                raise ValueError("Unknown FFN dimension")
-        
-        if hasattr(model_config, "num_local_experts"):
-            num_experts = model_config.num_local_experts
-        elif hasattr(model_config, "num_experts"):
-            num_experts = model_config.num_experts
-        else:
-            num_experts = 1
-            
-        ffn_params = n_layers * d_ff * linear_count * d_model
-        
-        shared_params = model_size_param * 1e9 - num_experts * ffn_params
-
-        model_size = shared_params + n_experts_per_tok * ffn_params
-
-        per_token_kv_size = 2 * n_layers * d_model * precision_bytes
-        
-        peak_bw_single = get_peak_bw(get_gpu_details())
-        peak_bw = peak_bw_single * get_gpu_number()
-        
-        context_prefill_size = context_length
-        kv_size = context_prefill_size * per_token_kv_size + (output_length - 1) * per_token_kv_size / 2
-        
-        kv_size = kv_size / 1e9
-        
-        n_vocab = model_config.vocab_size
-
-        end_to_end_time = (end - start) / batch_size
-        prefilling_time = stop_watch.prefilling_time / batch_size
-        decoding_time = stop_watch.decoding_time / batch_size
-        token_per_sec = output_length / decoding_time
-        achieve_mem_bw = (model_size * precision_bytes / 1e9 + kv_size) * token_per_sec
-        
-        avg_context_length = context_length + (output_length - 1) / 2
-        flops_per_token = 2 * model_size + ((linear_count + element_wise_mul) * n_layers * avg_context_length * d_model) + 4 * d_model + 2 * d_model * n_vocab
-        peak_flops_single = get_peak_flops(get_gpu_details(), self.precision)
-        peak_flops = peak_flops_single * get_gpu_number()
-        
-        ## TODO only support llama-type decoder only models and moe models of switch transformer and mixtrial
-        mfu = token_per_sec * flops_per_token / peak_flops
-        mbu = achieve_mem_bw / peak_bw
-        
-        print(f"mfu: {mfu}, mbu: {mbu}")
-        
-        return res, end_to_end_time, prefilling_time, token_per_sec, mfu, mbu
-
-    def generate_until(
-        self, requests: List[Instance], disable_tqdm: bool = False
-    ) -> List[str]:
-        res = []
-
-        def _collate(req: Tuple[str, dict]):
-            """Defines the key for the sorted method"""
-            # the negative sign on len(toks) sorts descending - this has a few advantages:
-            # - time estimates will always be over not underestimates, which is more useful for planning
-            # - to know the size of a batch when going through the list, you know the first one is always the batch
-            #   padded context length. this is useful to simplify the batching logic and more importantly to make
-            #   automatic adaptive batches much much easier to implement
-            # - any OOMs will happen right away rather than near the end
-            toks = self.tok_encode(req[0])
-            return -len(toks), req[0]
-
-        pbar = tqdm(
-            total=len(requests),
-            disable=(disable_tqdm or (self.rank != 0)),
-            desc="Running generate_until requests",
-        )
-        adaptive_batch_size = None
-        if self.batch_size == "auto":
-            # using rolling window with maximum context
-            print("Passed argument batch_size = auto. Detecting largest batch size")
-            batch_size = self._detect_batch_size()
-            print(f"Determined Largest batch size: {batch_size}")
-            adaptive_batch_size = batch_size
-        # for each different set of kwargs, we execute all requests, by batch.
-        batch_size = (
-            self.batch_size
-            if self.batch_size != "auto"
-            else adaptive_batch_size
-            if adaptive_batch_size is not None
-            else 0
-        )
-        batch_fn = (
-            self._batch_scheduler
-            if self.batch_size == "auto" and not adaptive_batch_size
-            else None
-        )
-
-        # we group requests by their generation_kwargs,
-        # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
-        # in the same batch.
-        # group_fn=lambda x: x[1] -> x=(context, gen_kwargs)
-        re_ords = Collator(
-            [reg.args for reg in requests],
-            sort_fn=_collate,
-            group_by="gen_kwargs",
-            group_fn=lambda x: x[1],
-        )
-        chunks = re_ords.get_batched(n=batch_size, batch_fn=batch_fn)
-        for chunk in chunks:
-            contexts, all_gen_kwargs = zip(*chunk)
-            # we assume all gen kwargs in the batch are the same
-            # this is safe to assume because the `grouper` object ensures it.
-            gen_kwargs = all_gen_kwargs[0]
-            # unpack our keyword arguments.
-            until = None
-            if isinstance(gen_kwargs, dict):
-                kwargs = copy.deepcopy(gen_kwargs)  # edge case for repeats > 1
-                if "until" in kwargs.keys():
-                    until = kwargs.pop("until")
-                    if isinstance(until, str):
-                        until = [kwargs]
-                    elif not isinstance(until, list):
-                        raise ValueError(
-                            f"Expected `kwargs['until']` to be of type Union[str,list] but got {until}"
-                        )
-            else:
-                raise ValueError(
-                    f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}"
-                )
-            # add EOS token to stop sequences
-            eos = "<|eot_id|>"
-            if not until:
-                until = [eos]
-            else:
-                until.append(eos)
-            
-            # is_gsm8k = kwargs.get("is_gsm8k", False)
-            # if is_gsm8k:
-            #     until = ["Question:", "Question", "</s>"]
-            #     eos_ids = [self.tokenizer.eos_token_id, 
-            #              self.tokenizer.convert_tokens_to_ids("<|eot_id|>")]
-                
-                    
-            if "max_gen_toks" in kwargs.keys():
-                max_gen_toks = kwargs.pop("max_gen_toks")
-            else:
-                max_gen_toks = self.max_gen_toks
-
-            # set the max length in tokens of inputs ("context_enc")
-            if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM:
-                # max len for inputs = max length, minus room to generate the max new tokens
-                max_ctx_len = self.max_length - max_gen_toks
-            elif self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM:
-                # max len for inputs = encoder's whole max_length
-                max_ctx_len = self.max_length
-
-            # encode, pad, and truncate contexts for this batch
-            context_enc, attn_masks = self.tok_batch_encode(
-                contexts,
-                left_truncate_len=max_ctx_len,
-                truncation=self.truncation,
-            )
-            
-            # print("context: ", self.tok_decode(context_enc[0]))
-            context_enc = context_enc.to(self.device)
-            attn_masks = attn_masks.to(self.device)
-
-            if "max_tokens" not in kwargs:
-                kwargs["max_tokens"] = max_gen_toks
-
-            # perform batched generation
-            cont, end_to_end_time, prefilling_time, token_per_sec, mfu, mbu = self._model_generate(
-                context=context_enc,
-                attention_mask=attn_masks,
-                stop=until,
-                **kwargs,
-            )
-
-            cont_toks_list = cont.tolist()
-            for cont_toks, context in zip(cont_toks_list, contexts):
-                # discard context + left-padding toks if using causal decoder-only LM
-                if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM:
-                    # print("After Generation: ", self.tok_decode(cont_toks))
-                    cont_toks = cont_toks[context_enc.shape[1] :]
-                
-                s = self.tok_decode(cont_toks)
-
-                # # use secondary stop seqs to cut off should-have-been-stopped content post-hoc
-                # if not is_gsm8k:
-                for term in until:
-                    if len(term) > 0:
-                        # ignore '' separator,
-                        # for seq2seq case where self.tok_decode(self.eot_token_id) = ''
-                        s = s.split(term)[0]
-                
-                # print(s)
-                res.append((s, end_to_end_time, prefilling_time, token_per_sec, mfu, mbu))
-
-                self.cache_hook.add_partial("generate_until", (context, gen_kwargs), s)
-                pbar.update(1)
-        # reorder this group of results back to original unsorted form
-        res = re_ords.get_original(res)
-
-        pbar.close()
-
-        return res

open-moe-llm-leaderboard-gh/src/backend/huggingface_generate_until.py

@@ -1,60 +0,0 @@
-from typing import List, Literal, Optional, Tuple, Union
-import torch
-import transformers
-
-from lm_eval.api.registry import register_model
-
-from src.backend.hflm_with_measurement import HFLMWithMeasurement
-
-
-@register_model("hf-chat")
-class HFLMwithChatTemplate(HFLMWithMeasurement):
-    def __init__(self, use_chat_template=True, **kwargs):
-        super().__init__(**kwargs)
-        self.use_chat_template = use_chat_template
-
-    def tok_batch_encode(
-        self,
-        strings: List[str],
-        padding_side: str = "left",
-        left_truncate_len: int = None,
-        truncation: bool = False,
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-
-        if self.use_chat_template:
-            try:
-                updated_strings = []
-                for input_string in strings:
-                    messages = [
-                        {"role": "user", "content": f"{input_string}"},
-                    ]
-                    if "dbrx" in self.model.name_or_path:
-                        updated_string = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-                    else:
-                        updated_string = self.tokenizer.apply_chat_template(messages, tokenize=False)
-                    updated_strings.append(updated_string)
-                strings = updated_strings[:]
-            except:
-                print(f"failed to update input string with chat template: {self._model}")
-        # encode a batch of strings. converts to tensors and pads automatically, unlike tok_encode.
-        old_padding_side = self.tokenizer.padding_side
-        self.tokenizer.padding_side = padding_side
-
-        if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM:
-            add_special_tokens = False
-        elif self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM:
-            add_special_tokens = True
-
-        encoding = self.tokenizer(
-            strings,
-            truncation=truncation,
-            padding="longest",
-            return_tensors="pt",
-            add_special_tokens=add_special_tokens,
-        )
-        if left_truncate_len:
-            encoding["input_ids"] = encoding["input_ids"][:, -left_truncate_len:]
-            encoding["attention_mask"] = encoding["attention_mask"][:, -left_truncate_len:]
-        self.tokenizer.padding_side = old_padding_side
-
-        return encoding["input_ids"], encoding["attention_mask"]

open-moe-llm-leaderboard-gh/src/backend/manage_requests.py

@@ -1,134 +0,0 @@
-import glob
-import json
-from dataclasses import dataclass
-from typing import Optional
-
-from huggingface_hub import HfApi, snapshot_download
-
-from src.utils import my_snapshot_download
-
-
-@dataclass
-class EvalRequest:
-    model: str
-    private: bool
-    status: str
-    json_filepath: str
-    weight_type: str = "Original"
-    model_type: str = ""  # pretrained, finetuned, with RL
-    inference_framework: str = "hf-chat"
-    precision: str = ""  # float16, bfloat16
-    base_model: Optional[str] = None  # for adapter models
-    revision: str = "main"  # commit
-    submitted_time: Optional[str] = (
-        "2022-05-18T11:40:22.519222"  # random date just so that we can still order requests by date
-    )
-    model_type: Optional[str] = None
-    likes: Optional[int] = 0
-    params: Optional[int] = None
-    license: Optional[str] = ""
-    batch_size: Optional[int] = 1
-    gpu_type: Optional[str] = "NVIDIA-A100-PCIe-80GB"
-
-    def get_model_args(self) -> str:
-        model_args = f"pretrained={self.model},revision={self.revision},parallelize=True"  # ,max_length=4096"
-        model_args += ",trust_remote_code=True,device_map=auto"
-        if self.precision in ["float16", "float32", "bfloat16"]:
-            model_args += f",dtype={self.precision}"
-            # Quantized models need some added config, the install of bits and bytes, etc
-            # elif self.precision == "8bit":
-            #    model_args += ",load_in_8bit=True"
-        elif self.precision == "4bit":
-           model_args += ",load_in_4bit=True"
-            # elif self.precision == "GPTQ":
-            # A GPTQ model does not need dtype to be specified,
-            # it will be inferred from the config
-        elif self.precision == "8bit":
-            model_args += ",load_in_8bit=True"
-        else:
-            raise Exception(f"Unknown precision {self.precision}.")
-        return model_args
-
-
-def set_eval_request(api: HfApi, eval_request: EvalRequest, set_to_status: str, hf_repo: str, local_dir: str):
-    """Updates a given eval request with its new status on the hub (running, completed, failed, ...)"""
-    json_filepath = eval_request.json_filepath
-
-    with open(json_filepath) as fp:
-        data = json.load(fp)
-
-    data["status"] = set_to_status
-
-    with open(json_filepath, "w") as f:
-        f.write(json.dumps(data))
-
-    api.upload_file(
-        path_or_fileobj=json_filepath,
-        path_in_repo=json_filepath.replace(local_dir, ""),
-        repo_id=hf_repo,
-        repo_type="dataset",
-    )
-
-
-def get_eval_requests(job_status: list, local_dir: str, hf_repo: str, do_download: bool = True) -> list[EvalRequest]:
-    """Get all pending evaluation requests and return a list in which private
-    models appearing first, followed by public models sorted by the number of
-    likes.
-
-    Returns:
-        `list[EvalRequest]`: a list of model info dicts.
-    """
-    if do_download:
-        my_snapshot_download(
-            repo_id=hf_repo, revision="main", local_dir=local_dir, repo_type="dataset", max_workers=60
-        )
-
-    json_files = glob.glob(f"{local_dir}/**/*.json", recursive=True)
-
-    eval_requests = []
-    for json_filepath in json_files:
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-        if data["status"] in job_status:
-            # import pdb
-            # breakpoint()
-            data["json_filepath"] = json_filepath
-
-            if "job_id" in data:
-                del data["job_id"]
-
-            eval_request = EvalRequest(**data)
-            eval_requests.append(eval_request)
-
-    return eval_requests
-
-
-def check_completed_evals(
-    api: HfApi,
-    hf_repo: str,
-    local_dir: str,
-    checked_status: str,
-    completed_status: str,
-    failed_status: str,
-    hf_repo_results: str,
-    local_dir_results: str,
-):
-    """Checks if the currently running evals are completed, if yes, update their status on the hub."""
-    my_snapshot_download(
-        repo_id=hf_repo_results, revision="main", local_dir=local_dir_results, repo_type="dataset", max_workers=60
-    )
-
-    running_evals = get_eval_requests([checked_status], hf_repo=hf_repo, local_dir=local_dir)
-
-    for eval_request in running_evals:
-        model = eval_request.model
-        print("====================================")
-        print(f"Checking {model}")
-
-        output_path = model
-        output_file = f"{local_dir_results}/{output_path}/results*.json"
-        output_file_exists = len(glob.glob(output_file)) > 0
-
-        if output_file_exists:
-            print(f"EXISTS output file exists for {model} setting it to {completed_status}")
-            set_eval_request(api, eval_request, completed_status, hf_repo, local_dir)

open-moe-llm-leaderboard-gh/src/backend/moe_infinity.py

@@ -1,124 +0,0 @@
-import torch
-import os
-import shutil
-from transformers import AutoTokenizer
-from transformers import AutoModelForCausalLM
-from moe_infinity import MoE
-from typing import List, Tuple, Optional, Union
-
-from lm_eval.api.registry import register_model
-
-from src.backend.hflm_with_measurement import HFLMWithMeasurement
-
-
-@register_model("moe-infinity")
-class MoEHFLM(HFLMWithMeasurement):
-    def __init__(
-        self,
-        pretrained: str = "mistralai/Mixtral-8x7B-Instruct-v0.1",
-        moe_config: dict = None,
-        offload_path=os.path.expanduser("~"),
-        device_memory_ratio=0.75,
-        use_chat_template=True,
-        *args,
-        **kwargs,
-    ):
-        # Initialize parent class without calling _create_model in the parent's __init__
-        self.checkpoint = pretrained
-        self.moe_config = moe_config if moe_config is not None else {}
-        self.offload_path = offload_path
-        self.device_memory_ratio = device_memory_ratio
-        self.use_chat_template = use_chat_template
-        if "device" in kwargs:
-            kwargs.pop("device")
-        kwargs["device_map"] = "cuda:0"
-        super().__init__(
-            *args, **kwargs, pretrained=pretrained
-        )  # Assuming HFLM accepts a 'pretrained' arg and handles it
-        # self._create_model()
-        shutil.rmtree(os.path.join(self.offload_path, "moe-infinity-offloads"))
-
-    def __del__(self):
-        # Clean up offloaded models from self.offload_path
-        shutil.rmtree(os.path.join(self.offload_path, "moe-infinity-offloads"))
-
-    def _create_model(self, *args, **kwargs):
-        """
-        Initializes the MoE model from MoE-infinity with the provided configuration.
-        """
-        # Ensure default configurations are set if not provided
-        default_moe_config = {
-            "offload_path": os.path.join(self.offload_path, "moe-infinity-offloads"),
-            "device_memory_ratio": self.device_memory_ratio,  # Default value, adjust as necessary
-        }
-        # Update default config with any user-provided config
-        final_moe_config = {**default_moe_config, **self.moe_config}
-
-        # dirty fix, to be removed when MoE-infinity supports move input to correct device
-        def MoEGenDecorator(func):
-            def wrapper(*args, **kwargs):
-                # Ensure all tensor in the input are in the same device as the model
-                args = [arg.to("cuda:0") if isinstance(arg, torch.Tensor) else arg for arg in args]
-                kwargs = {k: v.to("cuda:0") if isinstance(v, torch.Tensor) else v for k, v in kwargs.items()}
-                return func(*args, **kwargs)
-            return wrapper
-
-        self._model = MoE(self.checkpoint, final_moe_config)
-        self._model.generate = MoEGenDecorator(self._model.generate)
-        # self._model = AutoModelForCausalLM.from_pretrained(
-        #     self.checkpoint, torch_dtype=torch.float16, device_map="auto"
-        # )
-
-    @property
-    def max_length(self):
-        if self._max_length:  # if max length manually set, return it
-            return self._max_length
-        seqlen_config_attrs = ("n_positions", "max_position_embeddings", "n_ctx")
-        for attr in seqlen_config_attrs:
-            if hasattr(self.model.model.config, attr):
-                return getattr(self.model.model.config, attr)
-        if hasattr(self.tokenizer, "model_max_length"):
-            if self.tokenizer.model_max_length == 1000000000000000019884624838656:
-                return self._DEFAULT_MAX_LENGTH
-            return self.tokenizer.model_max_length
-        return self._DEFAULT_MAX_LENGTH
-
-    def tok_batch_encode(
-        self,
-        strings: List[str],
-        padding_side: str = "left",
-        left_truncate_len: int = None,
-        truncation: bool = False,
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-
-        if self.use_chat_template:
-            try:
-                updated_strings = []
-                for input_string in strings:
-                    messages = [
-                        {"role": "user", "content": f"{input_string}"},
-                    ]
-                    updated_string = self.tokenizer.apply_chat_template(messages, tokenize=False)
-                    updated_strings.append(updated_string)
-                strings = updated_strings[:]
-            except:
-                print(f"failed to update input string with chat template: {self._model}")
-        # encode a batch of strings. converts to tensors and pads automatically, unlike tok_encode.
-        old_padding_side = self.tokenizer.padding_side
-        self.tokenizer.padding_side = padding_side
-
-        add_special_tokens = False
-
-        encoding = self.tokenizer(
-            strings,
-            truncation=truncation,
-            padding="longest",
-            return_tensors="pt",
-            add_special_tokens=add_special_tokens,
-        )
-        if left_truncate_len:
-            encoding["input_ids"] = encoding["input_ids"][:, -left_truncate_len:]
-            encoding["attention_mask"] = encoding["attention_mask"][:, -left_truncate_len:]
-        self.tokenizer.padding_side = old_padding_side
-
-        return encoding["input_ids"], encoding["attention_mask"]

open-moe-llm-leaderboard-gh/src/backend/run_eval_suite.py

@@ -1,130 +0,0 @@
-from lm_eval import evaluator
-from lm_eval.tasks import TaskManager
-from lm_eval.api.metrics import mean
-from lm_eval.api.task import ConfigurableTask
-
-from src.backend.manage_requests import EvalRequest
-
-
-orig_process_results = ConfigurableTask.process_results
-orig_aggregation = ConfigurableTask.aggregation
-orig_higher_is_better = ConfigurableTask.higher_is_better
-
-def process_results_decorator(func):
-    def wrapper(self, doc, results, *args, **kwargs):
-        processed_results = [r[0] for r in results]
-
-        end_to_end_time = sum([r[1] for r in results]) / len(results)
-        prefilling_time = sum([r[2] for r in results]) / len(results)
-        decoding_throughput = sum([r[3] for r in results]) / len(results)
-        mfu = sum([r[4] for r in results]) / len(results)
-        mbu = sum([r[5] for r in results]) / len(results)
-        # print(f"end_to_end_time: {end_to_end_time}, prefilling_time: {prefilling_time}, decoding_throughput: {decoding_throughput}")
-
-        result_dict = func(self, doc, processed_results, *args, **kwargs)
-        result_dict["end_to_end_time"] = end_to_end_time
-        result_dict["prefilling_time"] = prefilling_time
-        result_dict["decoding_throughput"] = decoding_throughput
-        result_dict["mfu"] = mfu * 100
-        result_dict["mbu"] = mbu * 100
-        return result_dict
-    return wrapper
-ConfigurableTask.process_results = process_results_decorator(orig_process_results)
-
-def aggregation_decorator(func):
-    def wrapper(self, *args, **kwargs):
-        aggregation_list = func(self, *args, **kwargs)
-        aggregation_list["end_to_end_time"] = mean
-        aggregation_list["prefilling_time"] = mean
-        aggregation_list["decoding_throughput"] = mean
-        aggregation_list["mfu"] = mean
-        aggregation_list["mbu"] = mean
-        return aggregation_list
-    return wrapper
-ConfigurableTask.aggregation = aggregation_decorator(orig_aggregation)
-
-def higher_is_better_decorator(func):
-    def wrapper(self, *args, **kwargs):
-        higher_is_better_dict = func(self, *args, **kwargs)
-        higher_is_better_dict["end_to_end_time"] = False
-        higher_is_better_dict["prefilling_time"] = False
-        higher_is_better_dict["decoding_throughput"] = True
-        higher_is_better_dict["mfu"] = True
-        higher_is_better_dict["mbu"] = True
-        return higher_is_better_dict
-    return wrapper
-ConfigurableTask.higher_is_better = higher_is_better_decorator(orig_higher_is_better)
-
-# from src.backend.tasks.xsum.task import XSum
-# from src.backend.tasks.xsum.task_v2 import XSumv2
-
-# from src.backend.tasks.cnndm.task import CNNDM
-# from src.backend.tasks.cnndm.task_v2 import CNNDMv2
-
-from src.backend.tasks.selfcheckgpt.task import SelfCheckGPT
-
-from src.backend.huggingface_generate_until import HFLMwithChatTemplate
-from src.backend.moe_infinity import MoEHFLM
-
-def run_evaluation(
-    eval_request: EvalRequest,
-    task_names,
-    num_fewshot,
-    batch_size,
-    device,
-    use_cache=None,
-    limit=None,
-    max_nb_samples=100,
-) -> dict:
-    if limit:
-        print("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
-
-    # include_task_folder("src/backend/tasks/")
-    # initialize_tasks('INFO')
-
-    print(f"Allocating task manager for: {task_names}")
-
-    task_manager = TaskManager(include_path="./src/backend/tasks/")
-    # task_manager.initialize_tasks('INFO')
-
-    print(f"Considered Tasks: {task_names}")
-    # print(f"Allowed Tasks: {tasks.ALL_TASKS}")
-
-    # task_names = utils.pattern_match(task_names, tasks.ALL_TASKS)
-
-    print(f"Selected Tasks: {task_names}")
-    print(f"Eval Request: {eval_request}")
-    print(
-        f"Num Fewshot: {num_fewshot}, Batch Size: {batch_size}, Device: {device}, Use Cache: {use_cache}, Limit: {limit}"
-    )
-    # hf-chat is implemented to use apply_chat_template
-    results = evaluator.simple_evaluate(
-        model=eval_request.inference_framework,  # "hf-chat", "moe-infinity"
-        model_args=eval_request.get_model_args(),
-        tasks=task_names,
-        num_fewshot=num_fewshot,
-        batch_size=batch_size,
-        max_batch_size=8,
-        device=device,
-        use_cache=use_cache,
-        limit=limit,
-        write_out=True,
-        task_manager=task_manager,
-        verbosity="WARNING",
-    )
-
-    results["config"]["model_dtype"] = eval_request.precision
-    results["config"]["model_name"] = eval_request.model
-    results["config"]["model_sha"] = eval_request.revision
-    results["config"]["inference_framework"] = eval_request.inference_framework
-
-    if max_nb_samples is not None:
-        if "samples" in results:
-            samples = results["samples"]
-            for task_name in samples.keys():
-                if len(samples[task_name]) > max_nb_samples:
-                    results["samples"][task_name] = results["samples"][task_name][:max_nb_samples]
-
-    # print(evaluator.make_table(results))
-
-    return results

open-moe-llm-leaderboard-gh/src/backend/sort_queue.py

@@ -1,28 +0,0 @@
-from dataclasses import dataclass
-from huggingface_hub import HfApi
-from src.backend.manage_requests import EvalRequest
-
-
-@dataclass
-class ModelMetadata:
-    likes: int = 0
-    size: int = 15
-
-
-def sort_models_by_priority(api: HfApi, models: list[EvalRequest]) -> list[EvalRequest]:
-    private_models = [model for model in models if model.private]
-    public_models = [model for model in models if not model.private]
-
-    return sort_by_submit_date(private_models) + sort_by_submit_date(public_models)
-
-
-def sort_by_submit_date(eval_requests: list[EvalRequest]) -> list[EvalRequest]:
-    return sorted(eval_requests, key=lambda x: x.submitted_time, reverse=False)
-
-
-def sort_by_size(eval_requests: list[EvalRequest]) -> list[EvalRequest]:
-    return sorted(eval_requests, key=lambda x: x.params, reverse=False)
-
-
-def sort_by_likes(eval_requests: list[EvalRequest]) -> list[EvalRequest]:
-    return sorted(eval_requests, key=lambda x: x.likes, reverse=False)

open-moe-llm-leaderboard-gh/src/backend/tasks/__init__.py

@@ -1,7 +0,0 @@
-from src.backend.tasks.xsum.task import XSum
-from src.backend.tasks.xsum.task_v2 import XSumv2
-
-from src.backend.tasks.cnndm.task import CNNDM
-from src.backend.tasks.cnndm.task_v2 import CNNDMv2
-
-from src.backend.tasks.selfcheckgpt.task import SelfCheckGPT

open-moe-llm-leaderboard-gh/src/backend/tasks/cnndm/README.md

@@ -1,54 +0,0 @@
-# Task-name
-
-### Paper
-
-Title: `Know What You Don’t Know: Unanswerable Questions for SQuAD`
-Abstract: https://arxiv.org/abs/1806.03822
-
-Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset,
-consisting of questions posed by crowdworkers on a set of Wikipedia articles,
-where the answer to every question is a segment of text, or span, from the
-corresponding reading passage, or the question might be unanswerable.
-SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable
-questions written adversarially by crowdworkers to look similar to answerable ones.
-To do well on SQuAD2.0, systems must not only answer questions when possible, but
-also determine when no answer is supported by the paragraph and abstain from answering.
-
-Homepage: https://rajpurkar.github.io/SQuAD-explorer/
-
-
-### Citation
-
-```
-@misc{rajpurkar2018know,
-    title={Know What You Don't Know: Unanswerable Questions for SQuAD},
-    author={Pranav Rajpurkar and Robin Jia and Percy Liang},
-    year={2018},
-    eprint={1806.03822},
-    archivePrefix={arXiv},
-    primaryClass={cs.CL}
-}
-```
-
-### Groups and Tasks
-
-#### Groups
-
-* Not part of a group yet
-
-#### Tasks
-
-* `squadv2`: `Default squadv2 task`
-
-### Checklist
-
-For adding novel benchmarks/datasets to the library:
-* [ ] Is the task an existing benchmark in the literature?
-  * [ ] Have you referenced the original paper that introduced the task?
-  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
-
-
-If other tasks on this dataset are already supported:
-* [ ] Is the "Main" variant of this task clearly denoted?
-* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
-* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

open-moe-llm-leaderboard-gh/src/backend/tasks/cnndm/cnndm.yaml

@@ -1,2 +0,0 @@
-task: cnndm
-class: !function task.CNNDM

open-moe-llm-leaderboard-gh/src/backend/tasks/cnndm/cnndm_v2.yaml

@@ -1,2 +0,0 @@
-task: cnndm_v2
-class: !function task_v2.CNNDMv2

open-moe-llm-leaderboard-gh/src/backend/tasks/cnndm/task.py

@@ -1,218 +0,0 @@
-from lm_eval.api.task import ConfigurableTask
-from lm_eval.api.instance import Instance
-
-# from lm_eval.api.registry import register_task
-from lm_eval.api.metrics import mean
-
-import torch
-import sacrebleu
-from rouge_score import rouge_scorer, scoring
-
-
-def bleu(refs, preds):
-    """
-    Returns `t5` style BLEU scores. See the related implementation:
-    https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L41
-
-    :param refs:
-        A `list` of `list` of reference `str`s.
-    :param preds:
-        A `list` of predicted `str`s.
-    """
-    score = sacrebleu.corpus_bleu(
-        preds,
-        refs,
-        smooth_method="exp",
-        smooth_value=0.0,
-        force=False,
-        lowercase=False,
-        tokenize="intl",
-        use_effective_order=False,
-    ).score
-    return score
-
-
-def rouge(refs, preds):
-    """
-    Returns `t5` style ROUGE scores. See the related implementation:
-    https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L68
-
-    :param refs:
-        A `list` of reference `strs`.
-    :param preds:
-        A `list` of predicted `strs`.
-    """
-    rouge_types = ["rouge1", "rouge2", "rougeLsum"]
-    scorer = rouge_scorer.RougeScorer(rouge_types)
-    # Add newlines between sentences to correctly compute `rougeLsum`.
-
-    def _prepare_summary(summary):
-        summary = summary.replace(" . ", ".\n")
-        return summary
-
-    # Accumulate confidence intervals.
-    aggregator = scoring.BootstrapAggregator()
-    for ref, pred in zip(refs, preds):
-        ref = _prepare_summary(ref)
-        pred = _prepare_summary(pred)
-        aggregator.add_scores(scorer.score(ref, pred))
-    result = aggregator.aggregate()
-    return {type: result[type].mid.fmeasure * 100 for type in rouge_types}
-
-
-# @register_task("cnndm")
-class CNNDM(ConfigurableTask):
-    VERSION = 0
-    DATASET_PATH = "cnn_dailymail"
-    DATASET_NAME = "3.0.0"
-
-    def __init__(self):
-        super().__init__(config={"metadata": {"version": self.VERSION}})
-        self.factkb_tokenizer = None
-        self.factkb_model = None
-        self.bert_score = None
-
-    def maybe_init_factkb(self):
-        if self.factkb_tokenizer is None or self.factkb_model is None:
-            from transformers import AutoTokenizer, AutoModelForSequenceClassification
-
-            self.factkb_tokenizer = AutoTokenizer.from_pretrained(
-                "roberta-base", padding="max_length", truncation=True
-            )
-            self.factkb_model = AutoModelForSequenceClassification.from_pretrained(
-                "bunsenfeng/FactKB", num_labels=2, device_map="auto"
-            )
-
-    def maybe_init_bertscore(self):
-        if self.bert_score is None:
-            from evaluate import load
-
-            self.bert_score = load("bertscore")
-
-    def has_training_docs(self):
-        return True
-
-    def has_validation_docs(self):
-        return True
-
-    def has_test_docs(self):
-        return True
-
-    def training_docs(self):
-        return self.dataset["train"]
-
-    def validation_docs(self):
-        return self.dataset["validation"]
-
-    def test_docs(self):
-        return self.dataset["test"]
-
-    def doc_to_text(self, doc):
-        return f'Document: {doc["article"]}\nSummary:'
-
-    @staticmethod
-    def should_decontaminate():
-        return True
-
-    def doc_to_decontamination_query(self, doc):
-        return doc["article"]
-
-    def doc_to_target(self, doc):
-        return doc["highlights"]
-
-    def construct_requests(self, doc, ctx, **kwargs):
-        """Uses RequestFactory to construct Requests and returns an iterable of
-        Requests which will be sent to the LM.
-
-        :param doc:
-            The document as returned from training_docs, validation_docs, or test_docs.
-        :param ctx: str
-            The context string, generated by fewshot_context. This includes the natural
-            language description, as well as the few shot examples, and the question
-            part of the document for `doc`.
-        """
-
-        return [Instance(request_type="generate_until", doc=doc, arguments=(ctx, {"until": ["\n"]}), idx=0, **kwargs)]
-
-    def process_results(self, doc, results):
-        completion = results[0]
-        # true_refs, false_refs = doc["correct_answers"], doc["incorrect_answers"]
-        # all_refs = true_refs + false_refs
-
-        document = doc["article"]
-        gold_summary = doc["highlights"]
-
-        true_refs = [doc["highlights"]]
-        all_refs = true_refs
-
-        # ROUGE-N
-        rouge_scores = [rouge([ref], [completion]) for ref in all_refs]
-        # ROUGE-1
-        rouge1_scores = [score["rouge1"] for score in rouge_scores]
-        # ROUGE-2
-        rouge2_scores = [score["rouge2"] for score in rouge_scores]
-        # ROUGE-L
-        rougeL_scores = [score["rougeLsum"] for score in rouge_scores]
-
-        self.maybe_init_factkb()
-        input_factkb = [[completion, document]]
-        factkb_tokens = self.factkb_tokenizer(
-            input_factkb, return_tensors="pt", padding="max_length", truncation=True
-        ).to(self.factkb_model.device)
-        factkb_logits = self.factkb_model(**factkb_tokens).logits
-        factkb_res = torch.softmax(factkb_logits, dim=1)
-
-        self.maybe_init_bertscore()
-        bert_score_res = self.bert_score.compute(
-            predictions=[completion], references=[gold_summary], model_type="microsoft/deberta-xlarge-mnli", lang="en"
-        )
-
-        res = {
-            "rouge1": rouge1_scores[0],
-            "rouge2": rouge2_scores[0],
-            "rougeL": rougeL_scores[0],
-            "factKB": float(factkb_res[0][1]),
-            "bertscore_precision": float(bert_score_res["precision"][0]),
-            "bertscore_recall": float(bert_score_res["recall"][0]),
-            "bertscore_f1": float(bert_score_res["f1"][0]),
-        }
-
-        return res
-
-    def aggregation(self):
-        """
-        :returns: {str: [float] -> float}
-            A dictionary where keys are the names of submetrics and values are
-            functions that aggregate a list of metrics
-        """
-        return {
-            k: mean
-            for k in [
-                "rouge1",
-                "rouge2",
-                "rougeL",
-                "factKB",
-                "bertscore_precision",
-                "bertscore_recall",
-                "bertscore_f1",
-            ]
-        }
-
-    def higher_is_better(self):
-        """
-        :returns: {str: bool}
-            A dictionary where keys are the names of submetrics and values are
-            whether a higher value of the submetric is better
-        """
-        return {
-            k: True
-            for k in [
-                "rouge1",
-                "rouge2",
-                "rougeL",
-                "factKB",
-                "bertscore_precision",
-                "bertscore_recall",
-                "bertscore_f1",
-            ]
-        }

open-moe-llm-leaderboard-gh/src/backend/tasks/cnndm/task_v2.py

@@ -1,231 +0,0 @@
-from lm_eval.api.task import ConfigurableTask
-from lm_eval.api.instance import Instance
-
-# from lm_eval.api.registry import register_task
-from lm_eval.api.metrics import mean
-
-import torch
-import sacrebleu
-from rouge_score import rouge_scorer, scoring
-
-
-def bleu(refs, preds):
-    """
-    Returns `t5` style BLEU scores. See the related implementation:
-    https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L41
-
-    :param refs:
-        A `list` of `list` of reference `str`s.
-    :param preds:
-        A `list` of predicted `str`s.
-    """
-    score = sacrebleu.corpus_bleu(
-        preds,
-        refs,
-        smooth_method="exp",
-        smooth_value=0.0,
-        force=False,
-        lowercase=False,
-        tokenize="intl",
-        use_effective_order=False,
-    ).score
-    return score
-
-
-def rouge(refs, preds):
-    """
-    Returns `t5` style ROUGE scores. See the related implementation:
-    https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L68
-
-    :param refs:
-        A `list` of reference `strs`.
-    :param preds:
-        A `list` of predicted `strs`.
-    """
-    rouge_types = ["rouge1", "rouge2", "rougeLsum"]
-    scorer = rouge_scorer.RougeScorer(rouge_types)
-    # Add newlines between sentences to correctly compute `rougeLsum`.
-
-    def _prepare_summary(summary):
-        summary = summary.replace(" . ", ".\n")
-        return summary
-
-    # Accumulate confidence intervals.
-    aggregator = scoring.BootstrapAggregator()
-    for ref, pred in zip(refs, preds):
-        ref = _prepare_summary(ref)
-        pred = _prepare_summary(pred)
-        aggregator.add_scores(scorer.score(ref, pred))
-    result = aggregator.aggregate()
-    return {type: result[type].mid.fmeasure * 100 for type in rouge_types}
-
-
-# @register_task("cnndm_v2")
-class CNNDMv2(ConfigurableTask):
-    VERSION = 2
-    DATASET_PATH = "cnn_dailymail"
-    DATASET_NAME = "3.0.0"
-
-    def __init__(self):
-        super().__init__(
-            config={
-                "metadata": {"version": self.VERSION},
-                "generation_kwargs": {"do_sample": False, "temperature": 0.0, "until": ["\n", "\n\n"]},
-            }
-        )
-        self.factkb_tokenizer = None
-        self.factkb_model = None
-        self.bert_score = None
-
-    def maybe_init_factkb(self):
-        if self.factkb_tokenizer is None or self.factkb_model is None:
-            from transformers import AutoTokenizer, AutoModelForSequenceClassification
-
-            self.factkb_tokenizer = AutoTokenizer.from_pretrained(
-                "roberta-base", padding="max_length", truncation=True
-            )
-            self.factkb_model = AutoModelForSequenceClassification.from_pretrained(
-                "bunsenfeng/FactKB", num_labels=2, device_map="auto"
-            )
-
-    def maybe_init_bertscore(self):
-        if self.bert_score is None:
-            from evaluate import load
-
-            self.bert_score = load("bertscore")
-
-    def has_training_docs(self):
-        return True
-
-    def has_validation_docs(self):
-        return True
-
-    def has_test_docs(self):
-        return True
-
-    def training_docs(self):
-        return self.dataset["train"]
-
-    def validation_docs(self):
-        return self.dataset["validation"]
-
-    def test_docs(self):
-        return self.dataset["test"]
-
-    # def custom_prompt(self):
-    #     res = "Provide a summary of the provided article."
-    #     return res
-
-    # def fewshot_delimiter(self):
-    #     return "\n\n"
-
-    # From https://arxiv.org/abs/2305.14739
-    def doc_to_text(self, doc):
-        return f'Article: {doc["article"]}\nSummarize the article. Summary:'
-
-    @staticmethod
-    def should_decontaminate():
-        return True
-
-    def doc_to_decontamination_query(self, doc):
-        return doc["article"]
-
-    def doc_to_target(self, doc):
-        return doc["highlights"]
-
-    def construct_requests(self, doc, ctx, **kwargs):
-        """Uses RequestFactory to construct Requests and returns an iterable of
-        Requests which will be sent to the LM.
-
-        :param doc:
-            The document as returned from training_docs, validation_docs, or test_docs.
-        :param ctx: str
-            The context string, generated by fewshot_context. This includes the natural
-            language description, as well as the few shot examples, and the question
-            part of the document for `doc`.
-        """
-
-        return [Instance(request_type="generate_until", doc=doc, arguments=(ctx, {"until": ["\n"]}), idx=0, **kwargs)]
-
-    def process_results(self, doc, results):
-        completion = results[0]
-        # true_refs, false_refs = doc["correct_answers"], doc["incorrect_answers"]
-        # all_refs = true_refs + false_refs
-
-        document = doc["article"]
-        gold_summary = doc["highlights"]
-
-        true_refs = [doc["highlights"]]
-        all_refs = true_refs
-
-        # ROUGE-N
-        rouge_scores = [rouge([ref], [completion]) for ref in all_refs]
-        # ROUGE-1
-        rouge1_scores = [score["rouge1"] for score in rouge_scores]
-        # ROUGE-2
-        rouge2_scores = [score["rouge2"] for score in rouge_scores]
-        # ROUGE-L
-        rougeL_scores = [score["rougeLsum"] for score in rouge_scores]
-
-        self.maybe_init_factkb()
-        input_factkb = [[completion, document]]
-        factkb_tokens = self.factkb_tokenizer(
-            input_factkb, return_tensors="pt", padding="max_length", truncation=True
-        ).to(self.factkb_model.device)
-        factkb_logits = self.factkb_model(**factkb_tokens).logits
-        factkb_res = torch.softmax(factkb_logits, dim=1)
-
-        self.maybe_init_bertscore()
-        bert_score_res = self.bert_score.compute(
-            predictions=[completion], references=[gold_summary], model_type="microsoft/deberta-xlarge-mnli", lang="en"
-        )
-
-        res = {
-            "rouge1": rouge1_scores[0],
-            "rouge2": rouge2_scores[0],
-            "rougeL": rougeL_scores[0],
-            "factKB": float(factkb_res[0][1]),
-            "bertscore_precision": float(bert_score_res["precision"][0]),
-            "bertscore_recall": float(bert_score_res["recall"][0]),
-            "bertscore_f1": float(bert_score_res["f1"][0]),
-        }
-
-        return res
-
-    def aggregation(self):
-        """
-        :returns: {str: [float] -> float}
-            A dictionary where keys are the names of submetrics and values are
-            functions that aggregate a list of metrics
-        """
-        return {
-            k: mean
-            for k in [
-                "rouge1",
-                "rouge2",
-                "rougeL",
-                "factKB",
-                "bertscore_precision",
-                "bertscore_recall",
-                "bertscore_f1",
-            ]
-        }
-
-    def higher_is_better(self):
-        """
-        :returns: {str: bool}
-            A dictionary where keys are the names of submetrics and values are
-            whether a higher value of the submetric is better
-        """
-        return {
-            k: True
-            for k in [
-                "rouge1",
-                "rouge2",
-                "rougeL",
-                "factKB",
-                "bertscore_precision",
-                "bertscore_recall",
-                "bertscore_f1",
-            ]
-        }

open-moe-llm-leaderboard-gh/src/backend/tasks/faithdial/faithdial.yaml

@@ -1,14 +0,0 @@
-task: faithdial_hallu
-dataset_path: McGill-NLP/FaithDial
-training_split: train
-validation_split: validation
-test_split: test
-output_type: multiple_choice
-doc_to_text: !function utils.doc_to_text
-doc_to_target: !function utils.doc_to_target
-doc_to_choice: ["false", "true"]
-metric_list:
-  - metric: acc
-    higher_is_better: True
-metadata:
-  version: 0.0

open-moe-llm-leaderboard-gh/src/backend/tasks/faithdial/faithdial_v2.yaml

@@ -1,14 +0,0 @@
-task: faithdial_hallu_v2
-dataset_path: McGill-NLP/FaithDial
-training_split: train
-validation_split: validation
-test_split: test
-output_type: multiple_choice
-doc_to_text: !function utils.doc_to_text_v2
-doc_to_target: !function utils.doc_to_target
-doc_to_choice: ["false", "true"]
-metric_list:
-  - metric: acc
-    higher_is_better: True
-metadata:
-  version: 0.0

open-moe-llm-leaderboard-gh/src/backend/tasks/faithdial/utils.py

@@ -1,20 +0,0 @@
-from typing import List, Union
-
-ValueType = Union[str, List[str]]
-
-
-def doc_to_text(doc: dict[str, ValueType]) -> str:
-    history_str = " ".join([f'[{"Human" if i % 2 == 0 else "Assistant"}] {m}' for i, m in enumerate(doc["history"])])
-    doc_text = f'#Knowledge#: {doc["knowledge"]}\n#Dialogue History#: {history_str}\n#Response#: {doc["response"]}\n#Hallucinated#:'
-    return doc_text
-
-
-def doc_to_text_v2(doc: dict[str, ValueType]) -> str:
-    history_str = " ".join([f'[{"Human" if i % 2 == 0 else "Assistant"}] {m}' for i, m in enumerate(doc["history"])])
-    doc_text = f'#Knowledge#: {doc["knowledge"]}\n#Dialogue History#: {history_str}\n#Response#: {doc["original_response"]}\n#Hallucinated#:'
-    return doc_text
-
-
-def doc_to_target(doc: dict[str, ValueType]) -> str:
-    res = "true" if "Hallucination" in doc["BEGIN"] else "false"
-    return res

open-moe-llm-leaderboard-gh/src/backend/tasks/fever/fever10.yaml

@@ -1,16 +0,0 @@
-task: fever10
-dataset_path: fever
-dataset_name: v1.0
-output_type: multiple_choice
-training_split: train
-validation_split: labelled_dev
-test_split: null
-doc_to_text: "Claim: {{claim}}\nLabel:"
-doc_to_choice: ["SUPPORTS", "REFUTES", "NOT ENOUGH INFO"]
-doc_to_target: label
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true
-metadata:
-  version: 0.0

open-moe-llm-leaderboard-gh/src/backend/tasks/fever/fever11.yaml

@@ -1,16 +0,0 @@
-task: fever11
-dataset_path: pminervini/hl-fever
-dataset_name: v1.0
-output_type: multiple_choice
-training_split: train
-validation_split: dev
-test_split: null
-doc_to_text: "Claim: {{claim}}\nLabel:"
-doc_to_choice: ["supported", "refuted"]
-doc_to_target: label
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true
-metadata:
-  version: 0.0

open-moe-llm-leaderboard-gh/src/backend/tasks/gsm8k/gsm8k-custom.yaml

@@ -1,47 +0,0 @@
-group:
-  - math_word_problems
-task: gsm8k_custom
-dataset_path: gsm8k
-dataset_name: main
-output_type: generate_until
-training_split: train
-fewshot_split: train
-test_split: test
-doc_to_text: "Question: {{question}}\nAnswer:"
-doc_to_target: "{{answer}}" #" {{answer.split('### ')[-1].rstrip()}}"
-metric_list:
-  - metric: exact_match
-    aggregation: mean
-    higher_is_better: true
-    ignore_case: true
-    ignore_punctuation: false
-    regexes_to_ignore:
-      - ","
-      - "\\$"
-      - "(?s).*#### "
-      - "\\.$"
-generation_kwargs:
-  until:
-    - "Question:"
-    - "Question"
-    - "</s>"
-    - "<|im_end|>"
-  do_sample: false
-  temperature: 0.0
-  # is_gsm8k: true
-repeats: 1
-num_fewshot: 5
-filter_list:
-  - name: "strict-match"
-    filter:
-      - function: "regex"
-        regex_pattern: "#### (\\-?[0-9\\.\\,]+)"
-      - function: "take_first"
-  - name: "flexible-extract"
-    filter:
-      - function: "regex"
-        group_select: -1
-        regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)"
-      - function: "take_first"
-metadata:
-  version: 3.0

open-moe-llm-leaderboard-gh/src/backend/tasks/halueval/halueval_dialogue.yaml

@@ -1,29 +0,0 @@
-task: halueval_dialogue
-dataset_path: pminervini/HaluEval
-dataset_name: dialogue_samples
-output_type: generate_until
-training_split: null
-validation_split: null
-test_split: data
-num_fewshot: 0
-doc_to_text: !function utils.doc_to_text_dialogue
-doc_to_target: !function utils.doc_to_target
-process_results: !function utils.process_results
-generation_kwargs:
-  until:
-    - "\n"
-    - "."
-  do_sample: false
-  temperature: 0.0
-metric_list:
-  - metric: em
-    aggregation: mean
-    higher_is_better: true
-  - metric: correctness
-    aggregation: mean
-    higher_is_better: true
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true
-metadata:
-  version: 0.0

open-moe-llm-leaderboard-gh/src/backend/tasks/halueval/halueval_qa.yaml

@@ -1,29 +0,0 @@
-task: halueval_qa
-dataset_path: pminervini/HaluEval
-dataset_name: qa_samples
-output_type: generate_until
-training_split: null
-validation_split: null
-test_split: data
-num_fewshot: 0
-doc_to_text: !function utils.doc_to_text_qa
-doc_to_target: !function utils.doc_to_target
-process_results: !function utils.process_results
-generation_kwargs:
-  until:
-    - "\n"
-    - "."
-  do_sample: false
-  temperature: 0.0
-metric_list:
-  - metric: em
-    aggregation: mean
-    higher_is_better: true
-  - metric: correctness
-    aggregation: mean
-    higher_is_better: true
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true
-metadata:
-  version: 0.0

open-moe-llm-leaderboard-gh/src/backend/tasks/halueval/halueval_summarization.yaml

@@ -1,29 +0,0 @@
-task: halueval_summarization
-dataset_path: pminervini/HaluEval
-dataset_name: summarization_samples
-output_type: generate_until
-training_split: null
-validation_split: null
-test_split: data
-num_fewshot: 0
-doc_to_text: !function utils.doc_to_text_summarization
-doc_to_target: !function utils.doc_to_target
-process_results: !function utils.process_results
-generation_kwargs:
-  until:
-    - "\n"
-    - "."
-  do_sample: false
-  temperature: 0.0
-metric_list:
-  - metric: em
-    aggregation: mean
-    higher_is_better: true
-  - metric: correctness
-    aggregation: mean
-    higher_is_better: true
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true
-metadata:
-  version: 0.0

open-moe-llm-leaderboard-gh/src/backend/tasks/halueval/utils.py

@@ -1,154 +0,0 @@
-# Main reference: https://github.com/RUCAIBox/HaluEval/blob/main/evaluation/evaluate.py
-
-QA_INSTURCTIONS = """I want you act as an answer judge. Given a question and an answer, your objective is to determine if the provided answer contains non-factual or hallucinated information. You SHOULD give your judgement based on the following hallucination types and the world knowledge.
-
-You are trying to determine if the answer misunderstands the question context and intention.
-#Question#: What is a rare breed of dog that was derived as a variant of Rat Terrier, Shiloh Shepherd dog or American Hairless Terrier?
-#Answer#: American Hairless Terrier
-#Your Judgement#: No
-
-You are trying to determine if there is a factual contradiction between the answer and the world knowledge. Some information in the answer might be fabricated.
-#Question#: Are the New Orleans Outfall Canals the same length as the Augusta Canal?
-#Answer#: No, the New Orleans Outfall Canals and the Augusta Canal are not the same length. The Orleans Canal is approximately 3.6 miles (5.8 kilometers) long while the Augusta Canal is approximately 7 miles (11.3 kilometers) long.
-#Your Judgement#: Yes
-#Question#: What U.S Highway gives access to Zilpo Road, and is also known as Midland Trail?
-#Answer#: U.S Highway 70
-#Your Judgement#: Yes
-
-You are trying to determine if the answer is too general or too specific to answer the question at an appropriate level of specificity.
-#Question#: What genre do Superheaven and Oceansize belong to?
-#Answer#: Superheaven and Oceansize belong to the rock genre.
-#Your Judgement#: No
-#Question#: What profession do Kōbō Abe and Agatha Christie share?
-#Answer#: Playwright.
-#Your Judgement#: No
-
-You are trying to determine if the answer can be correctly inferred from the knowledge.
-#Question#: Which band has more members, Muse or The Raconteurs?
-#Answer#: Muse has more members than The Raconteurs.
-#Your Judgement#: Yes
-#Question#: Which is currently more valuable, Temagami-Lorrain Mine or Meadowbank Gold Mine?
-#Answer#: Meadowbank Gold Mine, since Meadowbank Gold Mine is still producing gold and the TemagamiLorrain Mine has been inactive for years.
-#Your Judgement#: No
-
-You should try your best to determine if the answer contains non-factual or hallucinated information according to the above hallucination types. The answer you give MUST be \"Yes\" or \"No\""."""
-
-DIALOGUE_INSTRUCTIONS = """I want you act as a response judge. Given a dialogue history and a response, your objective is to determine if the provided response contains non-factual or hallucinated information. You SHOULD give your judgement based on the following hallucination types and the world knowledge.
-
-You are trying to determine if the true entity in the response is replaced with a highly similar entity.
-#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
-#Response#: Christopher Nolan was the director. He also directed insomnia and inception.
-#Your Judgement#: No
-#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
-#Response#: Steven Spielberg was the director. He also directed insomnia and inception.
-#Your Judgement#: Yes
-
-You are trying to determine if the true entity in the response is replaced with a dissimilar entity.
-#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
-#Response#: Christopher Nolan was the director. He also directed insomnia and inception.
-#Your Judgement#: No
-#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
-#Response#: Batman Begins was the director. He also directed insomnia and inception.
-#Your Judgement#: Yes
-
-You are trying to determine if the true entity in the response is replaced with a dissimilar entity in a different entity type.
-#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
-#Response#: Christopher Nolan was the director. He also directed insomnia and inception.
-#Your Judgement#: No
-#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
-#Response#: United States of America was the director. He also directed insomnia and inception.
-#Your Judgement#: Yes
-
-You should try your best to determine if the response contains non-factual or hallucinated information according to the above hallucination types. The answer you give MUST be \"Yes\" or \"No\""."""
-
-SUMMARIZATION_INSTRUCTIONS = """I want you act as a summary judge. Given a document and a summary, your objective is to determine if the provided summary contains non-factual or hallucinated information. You SHOULD give your judgement based on the following hallucination types and the world knowledge.
-
-You are trying to determine if the summary is factual but some information cannot be directly inferred or entailed from the document.
-#Document#: The panther chameleon was found on Monday by a dog walker in the wooded area at Marl Park. It had to be put down after X-rays showed all of its legs were broken and it had a deformed spine. RSPCA Cymru said it was an "extremely sad example of an abandoned and neglected exotic pet". Inspector Selina Chan said: "It is a possibility that the owners took on this animal but were unable to provide the care he needs and decided to release him to the wild. "We are urging potential owners of exotic animals to thoroughly research what is required in the care of the particular species before taking one on. "Potential owners need to make sure they can give their animal the environment it needs and they have the facilities, time, financial means and long-term commitment to maintain a good standard of care, as required under the Animal Welfare Act 2006." She added it was illegal to release non-native species into the wild.
-#Summary#: A chameleon that was found in a Cardiff park has been put down after being abandoned and neglected by its owners.
-#Your Judgement#: Yes
-
-You are trying to determine if there exists some non-factual and incorrect information in the summary.  
-#Document#: The city was brought to a standstill on 15 December last year when a gunman held 18 hostages for 17 hours. Family members of victims Tori Johnson and Katrina Dawson were in attendance. Images of the floral tributes that filled the city centre in the wake of the siege were projected on to the cafe and surrounding buildings in an emotional twilight ceremony. Prime Minister Malcolm Turnbull gave an address saying a "whole nation resolved to answer hatred with love". "Testament to the spirit of Australians is that with such unnecessary, thoughtless tragedy, an amazing birth of mateship, unity and love occurs. Proud to be Australian," he said. How the Sydney siege unfolded New South Wales Premier Mike Baird has also announced plans for a permanent memorial to be built into the pavement in Martin Place. Clear cubes containing flowers will be embedded into the concrete and will shine with specialised lighting. It is a project inspired by the massive floral tributes that were left in the days after the siege. "Something remarkable happened here. As a city we were drawn to Martin Place. We came in shock and in sorrow but every step we took was with purpose," he said on Tuesday.
-#Summary#: Crowds have gathered in Sydney's Martin Place to honour the victims of the Lindt cafe siege, one year on.
-#Your Judgement#: No
-
-You are trying to determine if there is a factual contradiction between the summary and the document.
-#Document#: Christopher Huxtable, 34, from Swansea, had been missing since the collapse in February. His body was found on Wednesday and workers who carried out the search formed a guard of honour as it was driven from the site in the early hours of the morning. Ken Cresswell, 57, and John Shaw, 61, both from Rotherham, remain missing. The body of a fourth man, Michael Collings, 53, from Brotton, Teesside, was previously recovered from the site. Swansea East MP Carolyn Harris, who has been involved with the family since the incident, said they still did not know all the facts about the collapse. She said: "I feel very sad. My heart and my prayers go out to the family who have waited desperately for Christopher's body to be found. They can finally have closure, and say goodbye to him and grieve his loss. "But let's not forget that there's two other families who are still waiting for their loved ones to be returned." The building was due for demolition when it partially collapsed in February.
-#Summary#: The body of a man whose body was found at the site of the Swansea Bay Power Station collapse has been removed from the site.
-#Your Judgement#: Yes
-
-You should try your best to determine if the summary contains non-factual or hallucinated information according to the above hallucination types. The answer you give MUST be \"Yes\" or \"No\""."""
-
-
-def doc_to_text_qa(doc: dict[str, str]) -> str:
-    # prompt = instruction + "\n\n#Question#: " + question + "\n#Answer#: " + answer + "\n#Your Judgement#:"
-    doc_text = (
-        QA_INSTURCTIONS
-        + "\n\n#Knowledge#: "
-        + doc["knowledge"]
-        + "\n#Question#: "
-        + doc["question"]
-        + "\n#Answer#: "
-        + doc["answer"]
-        + "\n#Your Judgement#:"
-    )
-    return doc_text
-
-
-def doc_to_text_dialogue(doc: dict[str, str]) -> str:
-    # prompt = instruction + "\n\n#Dialogue History#: " + dialog + "\n#Response#: " + response + "\n#Your Judgement#:"
-    doc_text = (
-        DIALOGUE_INSTRUCTIONS
-        + "\n\n#Knowledge#: "
-        + doc["knowledge"]
-        + "\n#Dialogue History#: "
-        + doc["dialogue_history"]
-        + "\n#Response#: "
-        + doc["response"]
-        + "\n#Your Judgement#:"
-    )
-    return doc_text
-
-
-def doc_to_text_summarization(doc: dict[str, str]) -> str:
-    # prompt1 = instruction + "\n\n#Document#: " + document
-    # prompt2 = "\n#Summary#: " + summary + "\n#Your Judgement#:"
-    doc_text_1 = SUMMARIZATION_INSTRUCTIONS + "\n\n#Document#: " + doc["document"]
-    doc_text_2 = "\n#Summary#: " + doc["summary"] + "\n#Your Judgement#:"
-    doc_text = doc_text_1 + doc_text_2
-    return doc_text
-
-
-def doc_to_target(doc: dict[str, str]) -> str:
-    return doc["hallucination"]
-
-
-def compute_metrics(gold_answer: str, prediction: str) -> dict[str, float]:
-    is_correct = True
-
-    if ("Yes" in prediction and "No" in prediction) or ("Yes" not in prediction and "No" not in prediction):
-        is_correct = False
-    elif "Yes" in prediction:
-        prediction = "yes"
-    elif "No" in prediction:
-        prediction = "no"
-
-    is_exact = gold_answer == prediction
-
-    res = {"correctness": 1.0 if is_correct else 0.0}
-    if is_correct:
-        res["em"] = 1.0 if is_exact else 0.0
-
-    res["acc"] = 1.0 if (is_correct and is_exact) else 0.0
-
-    return res
-
-
-def process_results(doc: dict[str, str], results: list[str]) -> dict[str, float]:
-    # results is e.g., ['Yes']
-    gold_list = doc_to_target(doc)
-    # gold_list is e.g., 'yes'
-    prediction = results[0].strip().split("\n")[0]
-    scores = compute_metrics(gold_list, prediction)
-    return scores

open-moe-llm-leaderboard-gh/src/backend/tasks/measurement_task_utils.py

@@ -1,69 +0,0 @@
-import functools
-from lm_eval.api.metrics import mean
-
-
-def process_results_decorator(func):
-    # This decorator processes the results of a task before passing them to the original process_results function
-    @functools.wraps(func)
-    def wrapper(self, doc, results, *args, **kwargs):
-        # We process the results here
-        processed_results = [r[0] for r in results]
-        
-        end_to_end_time = sum([r[1] for r in results]) / len(results)
-        prefilling_time = sum([r[2] for r in results]) / len(results)
-        decoding_throughput = sum([r[3] for r in results]) / len(results)
-        mfu = sum([r[4] for r in results]) / len(results)
-        mbu = sum([r[5] for r in results]) / len(results)
-        
-        # print(f"end_to_end_time: {end_to_end_time}, prefilling_time: {prefilling_time}, decoding_throughput: {decoding_throughput}")
-
-        # Now call the original process_results with the processed results
-        result_dict = func(self, doc, processed_results, *args, **kwargs)
-        result_dict["end_to_end_time"] = end_to_end_time
-        result_dict["prefilling_time"] = prefilling_time
-        result_dict["decoding_throughput"] = decoding_throughput
-        result_dict["mfu"] = mfu
-        result_dict["mbu"] = mbu
-        return result_dict
-    return wrapper
-
-
-def aggregation_decorator(func):
-    @functools.wraps(func)
-    def wrapper(self, *args, **kwargs):
-        aggregation_list = func(self, *args, **kwargs)
-        aggregation_list["end_to_end_time"] = mean
-        aggregation_list["prefilling_time"] = mean
-        aggregation_list["decoding_throughput"] = mean
-        aggregation_list["mfu"] = mean
-        aggregation_list["mbu"] = mean
-        return aggregation_list
-    return wrapper
-
-
-def higher_is_better_decorator(func):
-    @functools.wraps(func)
-    def wrapper(self, *args, **kwargs):
-        higher_is_better_dict = func(self, *args, **kwargs)
-        higher_is_better_dict["end_to_end_time"] = False
-        higher_is_better_dict["prefilling_time"] = False
-        higher_is_better_dict["decoding_throughput"] = True
-        higher_is_better_dict["mfu"] = True
-        higher_is_better_dict["mbu"] = True
-        return higher_is_better_dict
-    return wrapper
-
-
-def measure_system_metrics(cls):
-    method_decorators = {
-        'process_results': [process_results_decorator],
-        'aggregation': [aggregation_decorator],
-        'higher_is_better': [higher_is_better_decorator],
-    }
-    for method_name, decorators in method_decorators.items():
-        if callable(getattr(cls, method_name, None)):
-            original_method = getattr(cls, method_name)
-            for decorator in reversed(decorators):
-                original_method = decorator(original_method)
-            setattr(cls, method_name, original_method)
-    return cls

open-moe-llm-leaderboard-gh/src/backend/tasks/memo-trap/memo-trap.yaml

@@ -1,19 +0,0 @@
-task: memo-trap
-dataset_path: pminervini/inverse-scaling
-dataset_name: memo-trap
-output_type: multiple_choice
-training_split: null
-validation_split: data
-test_split: null
-num_fewshot: 0
-doc_to_text: "{{prompt}}"
-doc_to_target: answer_index
-doc_to_choice: "{{classes}}"
-should_decontaminate: False
-doc_to_decontamination_query: prompt
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true
-metadata:
-  version: 0.0