Duplicate from HuggingFaceH4/open_llm_leaderboard

Co-authored-by: Edward Beeching <edbeeching@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,6 @@
+evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb

README.md

@@ -0,0 +1,14 @@
+---
+title: Open LLM Leaderboard
+emoji: 🏆
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.27.0
+app_file: app.py
+pinned: true
+license: apache-2.0
+duplicated_from: HuggingFaceH4/open_llm_leaderboard
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,491 @@
+import os
+import json
+from datetime import datetime, timezone
+
+import numpy as np
+import gradio as gr
+import pandas as pd
+
+from apscheduler.schedulers.background import BackgroundScheduler
+from content import *
+from huggingface_hub import Repository, HfApi
+from transformers import AutoConfig
+from utils import get_eval_results_dicts, make_clickable_model
+
+# clone / pull the lmeh eval data
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+LMEH_REPO = "HuggingFaceH4/lmeh_evaluations"
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", None))
+
+api = HfApi()
+
+
+def restart_space():
+    api.restart_space(repo_id="HuggingFaceH4/open_llm_leaderboard", token=H4_TOKEN)
+
+
+def get_all_requested_models(requested_models_dir):
+    depth = 1
+    file_names = []
+
+    for root, dirs, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            file_names.extend([os.path.join(root, file) for file in files])
+
+    return set([file_name.lower().split("./evals/")[1] for file_name in file_names])
+
+
+repo = None
+requested_models = None
+if H4_TOKEN:
+    print("Pulling evaluation requests and results.")
+    # try:
+    #     shutil.rmtree("./evals/")
+    # except:
+    #     pass
+
+    repo = Repository(
+        local_dir="./evals/",
+        clone_from=LMEH_REPO,
+        use_auth_token=H4_TOKEN,
+        repo_type="dataset",
+    )
+    repo.git_pull()
+
+    requested_models_dir = "./evals/eval_requests"
+    requested_models = get_all_requested_models(requested_models_dir)
+
+
+# parse the results
+BENCHMARKS = ["arc_challenge", "hellaswag", "hendrycks", "truthfulqa_mc"]
+METRICS = ["acc_norm", "acc_norm", "acc_norm", "mc2"]
+
+
+def load_results(model, benchmark, metric):
+    file_path = os.path.join("evals", model, f"{model}-eval_{benchmark}.json")
+    if not os.path.exists(file_path):
+        return 0.0, None
+
+    with open(file_path) as fp:
+        data = json.load(fp)
+    accs = np.array([v[metric] for k, v in data["results"].items()])
+    mean_acc = np.mean(accs)
+    return mean_acc, data["config"]["model_args"]
+
+
+COLS = [
+    "Model",
+    "Revision",
+    "Average ⬆️",
+    "ARC (25-shot) ⬆️",
+    "HellaSwag (10-shot) ⬆️",
+    "MMLU (5-shot) ⬆️",
+    "TruthfulQA (0-shot) ⬆️",
+    "model_name_for_query",  # dummy column to implement search bar (hidden by custom CSS)
+]
+TYPES = ["markdown", "str", "number", "number", "number", "number", "number", "str"]
+
+if not IS_PUBLIC:
+    COLS.insert(2, "8bit")
+    TYPES.insert(2, "bool")
+
+EVAL_COLS = ["model", "revision", "private", "8bit_eval", "is_delta_weight", "status"]
+EVAL_TYPES = ["markdown", "str", "bool", "bool", "bool", "str"]
+
+BENCHMARK_COLS = [
+    "ARC (25-shot) ⬆️",
+    "HellaSwag (10-shot) ⬆️",
+    "MMLU (5-shot) ⬆️",
+    "TruthfulQA (0-shot) ⬆️",
+]
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)
+
+
+def get_leaderboard_df():
+    if repo:
+        print("Pulling evaluation results for the leaderboard.")
+        repo.git_pull()
+
+    all_data = get_eval_results_dicts(IS_PUBLIC)
+
+    if not IS_PUBLIC:
+        gpt4_values = {
+            "Model": f'<a target="_blank" href=https://arxiv.org/abs/2303.08774 style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">gpt4</a>',
+            "Revision": "tech report",
+            "8bit": None,
+            "Average ⬆️": 84.3,
+            "ARC (25-shot) ⬆️": 96.3,
+            "HellaSwag (10-shot) ⬆️": 95.3,
+            "MMLU (5-shot) ⬆️": 86.4,
+            "TruthfulQA (0-shot) ⬆️": 59.0,
+            "model_name_for_query": "GPT-4",
+        }
+        all_data.append(gpt4_values)
+        gpt35_values = {
+            "Model": f'<a target="_blank" href=https://arxiv.org/abs/2303.08774 style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">gpt3.5</a>',
+            "Revision": "tech report",
+            "8bit": None,
+            "Average ⬆️": 71.9,
+            "ARC (25-shot) ⬆️": 85.2,
+            "HellaSwag (10-shot) ⬆️": 85.5,
+            "MMLU (5-shot) ⬆️": 70.0,
+            "TruthfulQA (0-shot) ⬆️": 47.0,
+            "model_name_for_query": "GPT-3.5",
+        }
+        all_data.append(gpt35_values)
+
+    base_line = {
+        "Model": "<p>Baseline</p>",
+        "Revision": "N/A",
+        "8bit": None,
+        "Average ⬆️": 25.0,
+        "ARC (25-shot) ⬆️": 25.0,
+        "HellaSwag (10-shot) ⬆️": 25.0,
+        "MMLU (5-shot) ⬆️": 25.0,
+        "TruthfulQA (0-shot) ⬆️": 25.0,
+        "model_name_for_query": "baseline",
+    }
+
+    all_data.append(base_line)
+
+    df = pd.DataFrame.from_records(all_data)
+    df = df.sort_values(by=["Average ⬆️"], ascending=False)
+    df = df[COLS]
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, BENCHMARK_COLS)]
+    return df
+
+
+def get_evaluation_queue_df():
+    if repo:
+        print("Pulling changes for the evaluation queue.")
+        repo.git_pull()
+
+    entries = [
+        entry
+        for entry in os.listdir("evals/eval_requests")
+        if not entry.startswith(".")
+    ]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join("evals/eval_requests", entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data["# params"] = "unknown"
+            data["model"] = make_clickable_model(data["model"])
+            data["revision"] = data.get("revision", "main")
+
+            all_evals.append(data)
+        else:
+            # this is a folder
+            sub_entries = [
+                e
+                for e in os.listdir(f"evals/eval_requests/{entry}")
+                if not e.startswith(".")
+            ]
+            for sub_entry in sub_entries:
+                file_path = os.path.join("evals/eval_requests", entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                # data["# params"] = get_n_params(data["model"])
+                data["model"] = make_clickable_model(data["model"])
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] == "PENDING"]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"] == "FINISHED"]
+    df_pending = pd.DataFrame.from_records(pending_list)
+    df_running = pd.DataFrame.from_records(running_list)
+    df_finished = pd.DataFrame.from_records(finished_list)
+    return df_finished[EVAL_COLS], df_running[EVAL_COLS], df_pending[EVAL_COLS]
+
+
+original_df = get_leaderboard_df()
+leaderboard_df = original_df.copy()
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df()
+
+
+def is_model_on_hub(model_name, revision) -> bool:
+    try:
+        config = AutoConfig.from_pretrained(model_name, revision=revision)
+        return True
+
+    except Exception as e:
+        print("Could not get the model config from the hub.")
+        print(e)
+        return False
+
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    is_8_bit_eval: bool,
+    private: bool,
+    is_delta_weight: bool,
+):
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    # check the model actually exists before adding the eval
+    if revision == "":
+        revision = "main"
+    if is_delta_weight and not is_model_on_hub(base_model, revision):
+        error_message = f'Base model "{base_model}" was not found on hub!'
+        print(error_message)
+        return f"<p style='color: red; font-size: 20px; text-align: center;'>{error_message}</p>"
+
+    if not is_model_on_hub(model, revision):
+        error_message = f'Model "{model}"was not found on hub!'
+        return f"<p style='color: red; font-size: 20px; text-align: center;'>{error_message}</p>"
+
+    print("adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "private": private,
+        "8bit_eval": is_8_bit_eval,
+        "is_delta_weight": is_delta_weight,
+        "status": "PENDING",
+        "submitted_time": current_time,
+    }
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    OUT_DIR = f"eval_requests/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{is_8_bit_eval}_{is_delta_weight}.json"
+
+    # Check for duplicate submission
+    if out_path.lower() in requested_models:
+        duplicate_request_message = "This model has been already submitted."
+        return f"<p style='color: orange; font-size: 20px; text-align: center;'>{duplicate_request_message}</p>"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    api.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path,
+        repo_id=LMEH_REPO,
+        token=H4_TOKEN,
+        repo_type="dataset",
+    )
+
+    success_message = "Your request has been submitted to the evaluation queue!"
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{success_message}</p>"
+
+
+def refresh():
+    leaderboard_df = get_leaderboard_df()
+    (
+        finished_eval_queue_df,
+        running_eval_queue_df,
+        pending_eval_queue_df,
+    ) = get_evaluation_queue_df()
+    return (
+        leaderboard_df,
+        finished_eval_queue_df,
+        running_eval_queue_df,
+        pending_eval_queue_df,
+    )
+
+
+def search_table(df, query):
+    filtered_df = df[df["model_name_for_query"].str.contains(query, case=False)]
+    return filtered_df
+
+
+custom_css = """
+#changelog-text {
+    font-size: 16px !important;
+}
+
+#changelog-text h2 {
+    font-size: 18px !important;
+}
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+#leaderboard-table {
+    margin-top: 15px
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+    width: 30%;
+}
+
+/* Hides the final column */
+table td:last-child,
+table th:last-child {
+    display: none;
+}
+
+
+/* Limit the width of the first column so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+"""
+
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Row():
+        with gr.Column():
+            with gr.Accordion("📙 Citation", open=False):
+                citation_button = gr.Textbox(
+                    value=CITATION_BUTTON_TEXT,
+                    label=CITATION_BUTTON_LABEL,
+                    elem_id="citation-button",
+                ).style(show_copy_button=True)
+        with gr.Column():
+            with gr.Accordion("✨ CHANGELOG", open=False):
+                changelog = gr.Markdown(CHANGELOG_TEXT, elem_id="changelog-text")
+
+    with gr.Box(elem_id="search-bar-table-box"):
+        search_bar = gr.Textbox(
+            placeholder="🔍 Search your model and press ENTER...",
+            show_label=False,
+            elem_id="search-bar",
+        )
+
+        leaderboard_table = gr.components.Dataframe(
+            value=leaderboard_df,
+            headers=COLS,
+            datatype=TYPES,
+            max_rows=5,
+            elem_id="leaderboard-table",
+        )
+
+        # Dummy leaderboard for handling the case when the user uses backspace key
+        hidden_leaderboard_table_for_search = gr.components.Dataframe(
+            value=original_df, headers=COLS, datatype=TYPES, max_rows=5, visible=False
+        )
+
+        search_bar.submit(
+            search_table,
+            [hidden_leaderboard_table_for_search, search_bar],
+            leaderboard_table,
+        )
+
+    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+    with gr.Accordion("✅ Finished Evaluations", open=False):
+        finished_eval_table = gr.components.Dataframe(
+            value=finished_eval_queue_df,
+            headers=EVAL_COLS,
+            datatype=EVAL_TYPES,
+            max_rows=5,
+        )
+    with gr.Accordion("🔄 Running Evaluation Queue", open=False):
+        running_eval_table = gr.components.Dataframe(
+            value=running_eval_queue_df,
+            headers=EVAL_COLS,
+            datatype=EVAL_TYPES,
+            max_rows=5,
+        )
+
+    with gr.Accordion("⏳ Pending Evaluation Queue", open=False):
+        pending_eval_table = gr.components.Dataframe(
+            value=pending_eval_queue_df,
+            headers=EVAL_COLS,
+            datatype=EVAL_TYPES,
+            max_rows=5,
+        )
+
+    refresh_button = gr.Button("Refresh")
+    refresh_button.click(
+        refresh,
+        inputs=[],
+        outputs=[
+            leaderboard_table,
+            finished_eval_table,
+            running_eval_table,
+            pending_eval_table,
+        ],
+    )
+
+    with gr.Accordion("Submit a new model for evaluation"):
+        with gr.Row():
+            with gr.Column():
+                model_name_textbox = gr.Textbox(label="Model name")
+                revision_name_textbox = gr.Textbox(label="revision", placeholder="main")
+
+            with gr.Column():
+                is_8bit_toggle = gr.Checkbox(
+                    False, label="8 bit eval", visible=not IS_PUBLIC
+                )
+                private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
+                is_delta_weight = gr.Checkbox(False, label="Delta weights")
+                base_model_name_textbox = gr.Textbox(label="base model (for delta)")
+
+        submit_button = gr.Button("Submit Eval")
+        submission_result = gr.Markdown()
+        submit_button.click(
+            add_new_eval,
+            [
+                model_name_textbox,
+                base_model_name_textbox,
+                revision_name_textbox,
+                is_8bit_toggle,
+                private,
+                is_delta_weight,
+            ],
+            submission_result,
+        )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=3600)
+scheduler.start()
+demo.queue(concurrency_count=40).launch()

content.py

@@ -0,0 +1,130 @@
+CHANGELOG_TEXT = f"""
+## [2023-06-05] 
+- Increase concurrent thread count to 40
+- Search models on ENTER
+
+## [2023-06-02] 
+- Add a typeahead search bar
+- Use webhooks to automatically spawn a new Space when someone opens a PR
+- Start recording `submitted_time` for eval requests
+- Limit column max-width
+
+## [2023-05-30] 
+- Add a citation button
+- Simplify Gradio layout
+
+## [2023-05-29] 
+- Auto-restart every hour for the latest results
+- Sync with the internal version (minor style changes)
+
+## [2023-05-24] 
+- Add a baseline that has 25.0 for all values
+- Add CHANGELOG
+
+## [2023-05-23] 
+- Fix a CSS issue that made the leaderboard hard to read in dark mode
+
+## [2023-05-22] 
+- Display a success/error message after submitting evaluation requests
+- Reject duplicate submission
+- Do not display results that have incomplete results 
+- Display different queues for jobs that are RUNNING, PENDING, FINISHED status 
+
+## [2023-05-15] 
+- Fix a typo: from "TruthQA" to "TruthfulQA"
+
+## [2023-05-10] 
+- Fix a bug that prevented auto-refresh
+
+## [2023-05-10] 
+- Release the leaderboard to public
+"""
+
+TITLE = """<h1 align="center" id="space-title">🤗 Open LLM Leaderboard</h1>"""
+
+INTRODUCTION_TEXT = f"""
+📐 With the plethora of large language models (LLMs) and chatbots being released week upon week, often with grandiose claims of their performance, it can be hard to filter out the genuine progress that is being made by the open-source community and which model is the current state of the art. The 🤗 Open LLM Leaderboard aims to track, rank and evaluate LLMs and chatbots as they are released. 
+
+🤗 A key advantage of this leaderboard is that anyone from the community can submit a model for automated evaluation on the 🤗 GPU cluster, as long as it is a 🤗 Transformers model with weights on the Hub. We also support evaluation of models with delta-weights for non-commercial licensed models, such as LLaMa.
+
+📈 We evaluate models on 4 key benchmarks from the <a href="https://github.com/EleutherAI/lm-evaluation-harness" target="_blank"> Eleuther AI Language Model Evaluation Harness </a>, a unified framework to test generative language models on a large number of different evaluation tasks:
+- <a href="https://arxiv.org/abs/1803.05457" target="_blank">  AI2 Reasoning Challenge </a> (25-shot) - a set of grade-school science questions.
+- <a href="https://arxiv.org/abs/1905.07830" target="_blank">  HellaSwag </a> (10-shot) - a test of commonsense inference, which is easy for humans (~95%) but challenging for SOTA models.
+- <a href="https://arxiv.org/abs/2009.03300" target="_blank">  MMLU </a>  (5-shot) - a test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.
+- <a href="https://arxiv.org/abs/2109.07958" target="_blank">  TruthfulQA </a> (0-shot) - a benchmark to measure whether a language model is truthful in generating answers to questions.
+
+We chose these benchmarks as they test a variety of reasoning and general knowledge across a wide variety of fields in 0-shot and few-shot settings.
+"""
+
+EVALUATION_QUEUE_TEXT = f"""
+# Evaluation Queue for the 🤗 Open LLM Leaderboard, these models will be automatically evaluated on the 🤗 cluster
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{open-llm-leaderboard,
+  author = {Edward Beeching, Sheon Han, Nathan Lambert, Nazneen Rajani, Omar Sanseviero, Lewis Tunstall, Thomas Wolf},
+  title = {Open LLM Leaderboard},
+  year = {2023},
+  publisher = {Hugging Face},
+  howpublished = "\url{https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard}"
+
+}
+@software{eval-harness,
+  author       = {Gao, Leo and
+                  Tow, Jonathan and
+                  Biderman, Stella and
+                  Black, Sid and
+                  DiPofi, Anthony and
+                  Foster, Charles and
+                  Golding, Laurence and
+                  Hsu, Jeffrey and
+                  McDonell, Kyle and
+                  Muennighoff, Niklas and
+                  Phang, Jason and
+                  Reynolds, Laria and
+                  Tang, Eric and
+                  Thite, Anish and
+                  Wang, Ben and
+                  Wang, Kevin and
+                  Zou, Andy},
+  title        = {A framework for few-shot language model evaluation},
+  month        = sep,
+  year         = 2021,
+  publisher    = {Zenodo},
+  version      = {v0.0.1},
+  doi          = {10.5281/zenodo.5371628},
+  url          = {https://doi.org/10.5281/zenodo.5371628}
+}
+@misc{clark2018think,
+      title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge}, 
+      author={Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
+      year={2018},
+      eprint={1803.05457},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+@misc{zellers2019hellaswag,
+      title={HellaSwag: Can a Machine Really Finish Your Sentence?}, 
+      author={Rowan Zellers and Ari Holtzman and Yonatan Bisk and Ali Farhadi and Yejin Choi},
+      year={2019},
+      eprint={1905.07830},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{hendrycks2021measuring,
+      title={Measuring Massive Multitask Language Understanding}, 
+      author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
+      year={2021},
+      eprint={2009.03300},
+      archivePrefix={arXiv},
+      primaryClass={cs.CY}
+}
+@misc{lin2022truthfulqa,
+      title={TruthfulQA: Measuring How Models Mimic Human Falsehoods}, 
+      author={Stephanie Lin and Jacob Hilton and Owain Evans},
+      year={2022},
+      eprint={2109.07958},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}"""
+

requirements.txt

@@ -0,0 +1,67 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+APScheduler==3.10.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.11.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+gradio==3.27.0
+gradio_client==0.1.3
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.2
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.0
+Pillow==9.5.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+requests==2.28.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+transformers==4.28.1
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+yarl==1.8.2

utils.py

@@ -0,0 +1,157 @@
+import os
+import shutil
+import numpy as np
+import gradio as gr
+from huggingface_hub import Repository, HfApi
+from transformers import AutoConfig, AutoModel
+import json
+from apscheduler.schedulers.background import BackgroundScheduler
+import pandas as pd
+import datetime
+import glob
+from dataclasses import dataclass
+from typing import List, Tuple, Dict
+
+# clone / pull the lmeh eval data
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+LMEH_REPO = "HuggingFaceH4/lmeh_evaluations"
+
+METRICS = ["acc_norm", "acc_norm", "acc_norm", "mc2"]
+BENCHMARKS = ["arc_challenge", "hellaswag", "hendrycks", "truthfulqa_mc"]
+BENCH_TO_NAME = {
+    "arc_challenge": "ARC (25-shot) ⬆️",
+    "hellaswag": "HellaSwag (10-shot) ⬆️",
+    "hendrycks": "MMLU (5-shot) ⬆️",
+    "truthfulqa_mc": "TruthfulQA (0-shot) ⬆️",
+}
+
+
+def make_clickable_model(model_name):
+    LLAMAS = [
+        "huggingface/llama-7b",
+        "huggingface/llama-13b",
+        "huggingface/llama-30b",
+        "huggingface/llama-65b",
+    ]
+    if model_name in LLAMAS:
+        model = model_name.split("/")[1]
+        return f'<a target="_blank" href="https://ai.facebook.com/blog/large-language-model-llama-meta-ai/" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model}</a>'
+
+    if model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+        link = "https://huggingface.co/" + "CarperAI/stable-vicuna-13b-delta"
+        return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">stable-vicuna-13b</a>'
+
+    if model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+        link = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+        return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">alpaca-13b</a>'
+
+    # remove user from model name
+    # model_name_show = ' '.join(model_name.split('/')[1:])
+
+    link = "https://huggingface.co/" + model_name
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+@dataclass
+class EvalResult:
+    eval_name: str
+    org: str
+    model: str
+    revision: str
+    is_8bit: bool
+    results: dict
+
+    def to_dict(self):
+        if self.org is not None:
+            base_model = f"{self.org}/{self.model}"
+        else:
+            base_model = f"{self.model}"
+        data_dict = {}
+
+        data_dict["eval_name"] = self.eval_name
+        data_dict["8bit"] = self.is_8bit
+        data_dict["Model"] = make_clickable_model(base_model)
+        # dummy column to implement search bar (hidden by custom CSS)
+        data_dict["model_name_for_query"] = base_model
+        data_dict["Revision"] = self.revision
+        data_dict["Average ⬆️"] = round(
+            sum([v for k, v in self.results.items()]) / 4.0, 1
+        )
+        # data_dict["# params"] = get_n_params(base_model)
+
+        for benchmark in BENCHMARKS:
+            if not benchmark in self.results.keys():
+                self.results[benchmark] = None
+
+        for k, v in BENCH_TO_NAME.items():
+            data_dict[v] = self.results[k]
+
+        return data_dict
+
+
+def parse_eval_result(json_filepath: str) -> Tuple[str, dict]:
+    with open(json_filepath) as fp:
+        data = json.load(fp)
+
+    path_split = json_filepath.split("/")
+    org = None
+    model = path_split[-4]
+    is_8bit = path_split[-2] == "8bit"
+    revision = path_split[-3]
+    if len(path_split) == 7:
+        # handles gpt2 type models that don't have an org
+        result_key = f"{path_split[-4]}_{path_split[-3]}_{path_split[-2]}"
+    else:
+        result_key = (
+            f"{path_split[-5]}_{path_split[-4]}_{path_split[-3]}_{path_split[-2]}"
+        )
+        org = path_split[-5]
+
+    eval_result = None
+    for benchmark, metric in zip(BENCHMARKS, METRICS):
+        if benchmark in json_filepath:
+            accs = np.array([v[metric] for k, v in data["results"].items()])
+            mean_acc = round(np.mean(accs) * 100.0, 1)
+            eval_result = EvalResult(
+                result_key, org, model, revision, is_8bit, {benchmark: mean_acc}
+            )
+
+    return result_key, eval_result
+
+
+def get_eval_results(is_public) -> List[EvalResult]:
+    json_filepaths = glob.glob(
+        "evals/eval_results/public/**/16bit/*.json", recursive=True
+    )
+    if not is_public:
+        json_filepaths += glob.glob(
+            "evals/eval_results/private/**/*.json", recursive=True
+        )
+        json_filepaths += glob.glob(
+            "evals/eval_results/private/**/*.json", recursive=True
+        )
+        json_filepaths += glob.glob(
+            "evals/eval_results/public/**/8bit/*.json", recursive=True
+        )  # include the 8bit evals of public models
+    eval_results = {}
+
+    for json_filepath in json_filepaths:
+        result_key, eval_result = parse_eval_result(json_filepath)
+        if result_key in eval_results.keys():
+            eval_results[result_key].results.update(eval_result.results)
+        else:
+            eval_results[result_key] = eval_result
+
+    eval_results = [v for k, v in eval_results.items()]
+
+    return eval_results
+
+
+def get_eval_results_dicts(is_public=True) -> List[Dict]:
+    eval_results = get_eval_results(is_public)
+
+    return [e.to_dict() for e in eval_results]
+
+
+eval_results_dict = get_eval_results_dicts()
+# print(eval_results_dict)