merge refactor

.gitignore

@@ -1,9 +1,10 @@
-evals/
+auto_evals/
 venv/
 __pycache__/
 .env
 .ipynb_checkpoints
 *ipynb
+.vscode/
 
 gpt_4_evals/
 human_evals/

app.py

@@ -7,19 +7,25 @@ import gradio as gr
 import numpy as np
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import HfApi, Repository
+from huggingface_hub import HfApi
 from transformers import AutoConfig
 
-from content import *
-from elo_utils import get_elo_plots, get_elo_results_dicts
-from utils import get_eval_results_dicts, make_clickable_model, get_window_url_params
+from src.auto_leaderboard.get_model_metadata import apply_metadata
+from src.assets.text_content import *
+from src.elo_leaderboard.load_results import get_elo_plots, get_elo_results_dicts
+from src.auto_leaderboard.load_results import get_eval_results_dicts, make_clickable_model
+from src.assets.hardcoded_evals import gpt4_values, gpt35_values, baseline
+from src.assets.css_html_js import custom_css, get_window_url_params
+from src.utils_display import AutoEvalColumn, EvalQueueColumn, EloEvalColumn, fields, styled_error, styled_warning, styled_message
+from src.init import load_all_info_from_hub
 
 # clone / pull the lmeh eval data
 H4_TOKEN = os.environ.get("H4_TOKEN", None)
 LMEH_REPO = "HuggingFaceH4/lmeh_evaluations"
 HUMAN_EVAL_REPO = "HuggingFaceH4/scale-human-eval"
 GPT_4_EVAL_REPO = "HuggingFaceH4/open_llm_leaderboard_oai_evals"
-IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", None))
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+ADD_PLOTS = False
 
 api = HfApi()
 
@@ -29,113 +35,25 @@ def restart_space():
         repo_id="HuggingFaceH4/open_llm_leaderboard", token=H4_TOKEN
     )
 
+auto_eval_repo, human_eval_repo, gpt_4_eval_repo, requested_models = load_all_info_from_hub(LMEH_REPO, HUMAN_EVAL_REPO, GPT_4_EVAL_REPO)
 
-def get_all_requested_models(requested_models_dir):
-    depth = 1
-    file_names = []
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default]
 
-    for root, dirs, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            file_names.extend([os.path.join(root, file) for file in files])
-
-    return set([file_name.lower().split("./evals/")[1] for file_name in file_names])
-
-
-repo = None
-requested_models = None
-if H4_TOKEN:
-    print("Pulling evaluation requests and results.")
-    # try:
-    #     shutil.rmtree("./evals/")
-    # except:
-    #     pass
-
-    repo = Repository(
-        local_dir="./evals/",
-        clone_from=LMEH_REPO,
-        use_auth_token=H4_TOKEN,
-        repo_type="dataset",
-    )
-    repo.git_pull()
-
-    requested_models_dir = "./evals/eval_requests"
-    requested_models = get_all_requested_models(requested_models_dir)
-
-human_eval_repo = None
-if H4_TOKEN and not os.path.isdir("./human_evals"):
-    print("Pulling human evaluation repo")
-    human_eval_repo = Repository(
-        local_dir="./human_evals/",
-        clone_from=HUMAN_EVAL_REPO,
-        use_auth_token=H4_TOKEN,
-        repo_type="dataset",
-    )
-    human_eval_repo.git_pull()
-
-gpt_4_eval_repo = None
-if H4_TOKEN and not os.path.isdir("./gpt_4_evals"):
-    print("Pulling GPT-4 evaluation repo")
-    gpt_4_eval_repo = Repository(
-        local_dir="./gpt_4_evals/",
-        clone_from=GPT_4_EVAL_REPO,
-        use_auth_token=H4_TOKEN,
-        repo_type="dataset",
-    )
-    gpt_4_eval_repo.git_pull()
-
-# parse the results
-BENCHMARKS = ["arc_challenge", "hellaswag", "hendrycks", "truthfulqa_mc"]
-METRICS = ["acc_norm", "acc_norm", "acc_norm", "mc2"]
-
-
-def load_results(model, benchmark, metric):
-    file_path = os.path.join("evals", model, f"{model}-eval_{benchmark}.json")
-    if not os.path.exists(file_path):
-        return 0.0, None
+if not IS_PUBLIC:
+    COLS.insert(2, AutoEvalColumn.is_8bit.name)
+    TYPES.insert(2, AutoEvalColumn.is_8bit.type)
 
-    with open(file_path) as fp:
-        data = json.load(fp)
-    accs = np.array([v[metric] for k, v in data["results"].items()])
-    mean_acc = np.mean(accs)
-    return mean_acc, data["config"]["model_args"]
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
+BENCHMARK_COLS = [c.name for c in [AutoEvalColumn.arc, AutoEvalColumn.hellaswag, AutoEvalColumn.mmlu, AutoEvalColumn.truthfulqa]]
 
-COLS = [
-    "Model",
-    "Revision",
-    "Average ⬆️",
-    "ARC (25-shot) ⬆️",
-    "HellaSwag (10-shot) ⬆️",
-    "MMLU (5-shot) ⬆️",
-    "TruthfulQA (0-shot) ⬆️",
-    "model_name_for_query",  # dummy column to implement search bar (hidden by custom CSS)
-]
-TYPES = ["markdown", "str", "number", "number", "number", "number", "number", "str"]
-
-if not IS_PUBLIC:
-    COLS.insert(2, "8bit")
-    TYPES.insert(2, "bool")
-
-EVAL_COLS = ["model", "revision", "private", "8bit_eval", "is_delta_weight", "status"]
-EVAL_TYPES = ["markdown", "str", "bool", "bool", "bool", "str"]
-
-BENCHMARK_COLS = [
-    "ARC (25-shot) ⬆️",
-    "HellaSwag (10-shot) ⬆️",
-    "MMLU (5-shot) ⬆️",
-    "TruthfulQA (0-shot) ⬆️",
-]
-
-ELO_COLS = [
-    "Model",
-    "GPT-4 (all)",
-    "Human (all)",
-    "Human (instruct)",
-    "Human (code-instruct)",
-]
-ELO_TYPES = ["markdown", "number", "number", "number", "number"]
-ELO_SORT_COL = "GPT-4 (all)"
+ELO_COLS = [c.name for c in fields(EloEvalColumn)]
+ELO_TYPES = [c.type for c in fields(EloEvalColumn)]
+ELO_SORT_COL = EloEvalColumn.gpt4.name
 
 
 def has_no_nan_values(df, columns):
@@ -147,54 +65,21 @@ def has_nan_values(df, columns):
 
 
 def get_leaderboard_df():
-    if repo:
+    if auto_eval_repo:
         print("Pulling evaluation results for the leaderboard.")
-        repo.git_pull()
+        auto_eval_repo.git_pull()
 
     all_data = get_eval_results_dicts(IS_PUBLIC)
 
     if not IS_PUBLIC:
-        gpt4_values = {
-            "Model": f'<a target="_blank" href=https://arxiv.org/abs/2303.08774 style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">gpt4</a>',
-            "Revision": "tech report",
-            "8bit": None,
-            "Average ⬆️": 84.3,
-            "ARC (25-shot) ⬆️": 96.3,
-            "HellaSwag (10-shot) ⬆️": 95.3,
-            "MMLU (5-shot) ⬆️": 86.4,
-            "TruthfulQA (0-shot) ⬆️": 59.0,
-            "model_name_for_query": "GPT-4",
-        }
         all_data.append(gpt4_values)
-        gpt35_values = {
-            "Model": f'<a target="_blank" href=https://arxiv.org/abs/2303.08774 style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">gpt3.5</a>',
-            "Revision": "tech report",
-            "8bit": None,
-            "Average ⬆️": 71.9,
-            "ARC (25-shot) ⬆️": 85.2,
-            "HellaSwag (10-shot) ⬆️": 85.5,
-            "MMLU (5-shot) ⬆️": 70.0,
-            "TruthfulQA (0-shot) ⬆️": 47.0,
-            "model_name_for_query": "GPT-3.5",
-        }
         all_data.append(gpt35_values)
 
-    base_line = {
-        "Model": "<p>Baseline</p>",
-        "Revision": "N/A",
-        "8bit": None,
-        "Average ⬆️": 25.0,
-        "ARC (25-shot) ⬆️": 25.0,
-        "HellaSwag (10-shot) ⬆️": 25.0,
-        "MMLU (5-shot) ⬆️": 25.0,
-        "TruthfulQA (0-shot) ⬆️": 25.0,
-        "model_name_for_query": "baseline",
-    }
-
-    all_data.append(base_line)
+    all_data.append(baseline)
+    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
 
     df = pd.DataFrame.from_records(all_data)
-    df = df.sort_values(by=["Average ⬆️"], ascending=False)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[COLS]
 
     # filter out if any of the benchmarks have not been produced
@@ -203,20 +88,21 @@ def get_leaderboard_df():
 
 
 def get_evaluation_queue_df():
-    if repo:
+    # todo @saylortwift: replace the repo by the one you created for the eval queue
+    if auto_eval_repo:
         print("Pulling changes for the evaluation queue.")
-        repo.git_pull()
+        auto_eval_repo.git_pull()
 
     entries = [
         entry
-        for entry in os.listdir("evals/eval_requests")
+        for entry in os.listdir("auto_evals/eval_requests")
         if not entry.startswith(".")
     ]
     all_evals = []
 
     for entry in entries:
         if ".json" in entry:
-            file_path = os.path.join("evals/eval_requests", entry)
+            file_path = os.path.join("auto_evals/eval_requests", entry)
             with open(file_path) as fp:
                 data = json.load(fp)
 
@@ -229,11 +115,11 @@ def get_evaluation_queue_df():
             # this is a folder
             sub_entries = [
                 e
-                for e in os.listdir(f"evals/eval_requests/{entry}")
+                for e in os.listdir(f"auto_evals/eval_requests/{entry}")
                 if not e.startswith(".")
             ]
             for sub_entry in sub_entries:
-                file_path = os.path.join("evals/eval_requests", entry, sub_entry)
+                file_path = os.path.join("auto_evals/eval_requests", entry, sub_entry)
                 with open(file_path) as fp:
                     data = json.load(fp)
 
@@ -305,13 +191,15 @@ leaderboard_df = original_df.copy()
 
 def is_model_on_hub(model_name, revision) -> bool:
     try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision)
-        return True
+        AutoConfig.from_pretrained(model_name, revision=revision)
+        return True, None
+    
+    except ValueError as e:
+        return False, "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard."
 
     except Exception as e:
-        print("Could not get the model config from the hub.")
-        print(e)
-        return False
+        print("Could not get the model config from the hub.: \n", e)
+        return False, "was not found on hub!"
 
 
 def add_new_eval(
@@ -327,14 +215,15 @@ def add_new_eval(
     # check the model actually exists before adding the eval
     if revision == "":
         revision = "main"
-    if is_delta_weight and not is_model_on_hub(base_model, revision):
-        error_message = f'Base model "{base_model}" was not found on hub!'
-        print(error_message)
-        return f"<p style='color: red; font-size: 20px; text-align: center;'>{error_message}</p>"
 
-    if not is_model_on_hub(model, revision):
-        error_message = f'Model "{model}"was not found on hub!'
-        return f"<p style='color: red; font-size: 20px; text-align: center;'>{error_message}</p>"
+    if is_delta_weight: 
+        base_model_on_hub, error = is_model_on_hub(base_model, revision)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    model_on_hub, error = is_model_on_hub(model, revision)
+    if not model_on_hub:
+        return styled_error(f'Model "{model}" {error}')
 
     print("adding new eval")
 
@@ -355,14 +244,13 @@ def add_new_eval(
         user_name = model.split("/")[0]
         model_path = model.split("/")[1]
 
-    OUT_DIR = f"eval_requests/{user_name}"
+    OUT_DIR = f"auto_evals/eval_requests/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
     out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{is_8_bit_eval}_{is_delta_weight}.json"
 
     # Check for duplicate submission
-    if out_path.lower() in requested_models:
-        duplicate_request_message = "This model has been already submitted."
-        return f"<p style='color: orange; font-size: 20px; text-align: center;'>{duplicate_request_message}</p>"
+    if out_path.split("eval_requests/")[1].lower() in requested_models:
+        return styled_warning("This model has been already submitted.")
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))
@@ -375,8 +263,7 @@ def add_new_eval(
         repo_type="dataset",
     )
 
-    success_message = "Your request has been submitted to the evaluation queue!"
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{success_message}</p>"
+    return styled_message("Your request has been submitted to the evaluation queue!")
 
 
 def refresh():
@@ -395,7 +282,7 @@ def refresh():
 
 
 def search_table(df, query):
-    filtered_df = df[df["model_name_for_query"].str.contains(query, case=False)]
+    filtered_df = df[df[AutoEvalColumn.dummy.name].str.contains(query, case=False)]
     return filtered_df
 
 
@@ -413,83 +300,6 @@ def change_tab(query_param):
         return gr.Tabs.update(selected=0)
 
 
-custom_css = """
-#changelog-text {
-    font-size: 16px !important;
-}
-
-#changelog-text h2 {
-    font-size: 18px !important;
-}
-
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#models-to-add-text {
-    font-size: 18px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#search-bar-table-box > div:first-child {
-    background: none;
-    border: none;
-}
- 
-#search-bar {
-    padding: 0px;
-    width: 30%;
-}
-
-/* Hides the final column */
-#llm-benchmark-tab-table table td:last-child,
-#llm-benchmark-tab-table table th:last-child {
-    display: none;
-}
-
-/* Limit the width of the first column so that names don't expand too much */
-table td:first-child,
-table th:first-child {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
-}
-
-#scale-logo .download {
-    display: none;
-}
-"""
-
-
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -518,30 +328,52 @@ with demo:
                         show_label=False,
                         elem_id="search-bar",
                     )
-
-                    leaderboard_table = gr.components.Dataframe(
-                        value=leaderboard_df,
-                        headers=COLS,
-                        datatype=TYPES,
-                        max_rows=5,
-                        elem_id="leaderboard-table",
-                    )
+                    with gr.Tabs(elem_classes="tab-buttons"):
+                        with gr.TabItem("Light View"):
+                            leaderboard_table_lite = gr.components.Dataframe(
+                                value=leaderboard_df[COLS_LITE],
+                                headers=COLS_LITE,
+                                datatype=TYPES_LITE,
+                                max_rows=None,
+                                elem_id="leaderboard-table-lite",
+                            )
+                        with gr.TabItem("Extended Model View"):
+                            leaderboard_table = gr.components.Dataframe(
+                                value=leaderboard_df,
+                                headers=COLS,
+                                datatype=TYPES,
+                                max_rows=None,
+                                elem_id="leaderboard-table",
+                            )
 
                     # Dummy leaderboard for handling the case when the user uses backspace key
                     hidden_leaderboard_table_for_search = gr.components.Dataframe(
                         value=original_df,
                         headers=COLS,
                         datatype=TYPES,
-                        max_rows=5,
+                        max_rows=None,
                         visible=False,
                     )
-
                     search_bar.submit(
                         search_table,
                         [hidden_leaderboard_table_for_search, search_bar],
                         leaderboard_table,
                     )
 
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search_lite = gr.components.Dataframe(
+                        value=original_df[COLS_LITE],
+                        headers=COLS_LITE,
+                        datatype=TYPES_LITE,
+                        max_rows=None,
+                        visible=False,
+                    )
+                    search_bar.submit(
+                        search_table,
+                        [hidden_leaderboard_table_for_search_lite, search_bar],
+                        leaderboard_table_lite,
+                    )
+
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
@@ -625,7 +457,7 @@ with demo:
                     gr.Markdown(HUMAN_GPT_EVAL_TEXT, elem_classes="markdown-text")
                 with gr.Column(scale=1):
                     gr.Image(
-                        "scale-hf-logo.png", elem_id="scale-logo", show_label=False
+                        "src/assets/scale-hf-logo.png", elem_id="scale-logo", show_label=False
                     )
             gr.Markdown("## No tie allowed")
             elo_leaderboard_table = gr.components.Dataframe(
@@ -660,22 +492,23 @@ with demo:
         tabs,
         _js=get_window_url_params,
     )
-    # with gr.Box():
-    #     visualization_title = gr.HTML(VISUALIZATION_TITLE)
-    #     with gr.Row():
-    #         with gr.Column():
-    #             gr.Markdown(f"#### Figure 1: {PLOT_1_TITLE}")
-    #             plot_1 = gr.Plot(plot_1, show_label=False)
-    #         with gr.Column():
-    #             gr.Markdown(f"#### Figure 2: {PLOT_2_TITLE}")
-    #             plot_2 = gr.Plot(plot_2, show_label=False)
-    #     with gr.Row():
-    #         with gr.Column():
-    #             gr.Markdown(f"#### Figure 3: {PLOT_3_TITLE}")
-    #             plot_3 = gr.Plot(plot_3, show_label=False)
-    #         with gr.Column():
-    #             gr.Markdown(f"#### Figure 4: {PLOT_4_TITLE}")
-    #             plot_4 = gr.Plot(plot_4, show_label=False)
+    if ADD_PLOTS:
+        with gr.Box():
+            visualization_title = gr.HTML(VISUALIZATION_TITLE)
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown(f"#### Figure 1: {PLOT_1_TITLE}")
+                    plot_1 = gr.Plot(plot_1, show_label=False)
+                with gr.Column():
+                    gr.Markdown(f"#### Figure 2: {PLOT_2_TITLE}")
+                    plot_2 = gr.Plot(plot_2, show_label=False)
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown(f"#### Figure 3: {PLOT_3_TITLE}")
+                    plot_3 = gr.Plot(plot_3, show_label=False)
+                with gr.Column():
+                    gr.Markdown(f"#### Figure 4: {PLOT_4_TITLE}")
+                    plot_4 = gr.Plot(plot_4, show_label=False)
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=3600)

src/assets/css_html_js.py

@@ -0,0 +1,87 @@
+custom_css = """
+#changelog-text {
+    font-size: 16px !important;
+}
+
+#changelog-text h2 {
+    font-size: 18px !important;
+}
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+#leaderboard-table {
+    margin-top: 15px
+}
+
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+    width: 30%;
+}
+
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+    display: none;
+}
+
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+
+#scale-logo .download {
+    display: none;
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/assets/hardcoded_evals.py

@@ -0,0 +1,38 @@
+from src.utils_display import AutoEvalColumn, model_hyperlink
+
+gpt4_values = {
+    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt4"),
+    AutoEvalColumn.revision.name: "tech report",
+    AutoEvalColumn.is_8bit.name: None,
+    AutoEvalColumn.average.name: 84.3,
+    AutoEvalColumn.arc.name: 96.3,
+    AutoEvalColumn.hellaswag.name:  95.3,
+    AutoEvalColumn.mmlu.name:  86.4,
+    AutoEvalColumn.truthfulqa.name:  59.0,
+    AutoEvalColumn.dummy.name: "GPT-4",
+}
+
+gpt35_values = {
+    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt3.5"),
+    AutoEvalColumn.revision.name: "tech report",
+    AutoEvalColumn.is_8bit.name: None,
+    AutoEvalColumn.average.name: 71.9,
+    AutoEvalColumn.arc.name: 85.2,
+    AutoEvalColumn.hellaswag.name:  85.5,
+    AutoEvalColumn.mmlu.name:  70.0,
+    AutoEvalColumn.truthfulqa.name:  47.0,
+    AutoEvalColumn.dummy.name: "GPT-3.5",
+}
+
+baseline = {
+    AutoEvalColumn.model.name: "<p>Baseline</p>",
+    AutoEvalColumn.revision.name: "N/A",
+    AutoEvalColumn.is_8bit.name: None,
+    AutoEvalColumn.average.name: 25.0,
+    AutoEvalColumn.arc.name: 25.0,
+    AutoEvalColumn.hellaswag.name:  25.0,
+    AutoEvalColumn.mmlu.name:  25.0,
+    AutoEvalColumn.truthfulqa.name:  25.0,
+    AutoEvalColumn.dummy.name: "baseline",
+}
+

content.py → src/assets/text_content.py

@@ -1,4 +1,8 @@
 CHANGELOG_TEXT = f"""
+## [2023-06-16]
+- Refactored code base
+- Added new columns: number of parameters, hub likes, license
+
 ## [2023-06-13] 
 - Adjust description for TruthfulQA
 
@@ -13,7 +17,7 @@ CHANGELOG_TEXT = f"""
 - Add a typeahead search bar
 - Use webhooks to automatically spawn a new Space when someone opens a PR
 - Start recording `submitted_time` for eval requests
-- Limit column max-width
+- Limit AutoEvalColumn max-width
 
 ## [2023-05-30] 
 - Add a citation button

src/auto_leaderboard/get_model_metadata.py

@@ -0,0 +1,54 @@
+import re
+from typing import List
+
+from src.utils_display import AutoEvalColumn
+
+from huggingface_hub import HfApi
+import huggingface_hub
+api = HfApi()
+
+
+def get_model_infos_from_hub(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        model_name = model_data["model_name_for_query"]
+        try:
+            model_info = api.model_info(model_name)
+        except huggingface_hub.utils._errors.RepositoryNotFoundError:
+            model_data[AutoEvalColumn.license.name] = None
+            model_data[AutoEvalColumn.likes.name] = None
+            model_data[AutoEvalColumn.params.name] = None
+            continue
+
+        model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
+        model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
+        model_data[AutoEvalColumn.params.name] = get_model_size(model_name, model_info)
+
+
+def get_model_license(model_info):
+    try:
+        return model_info.cardData["license"]
+    except Exception:
+        return None
+
+def get_model_likes(model_info):
+    return model_info.likes
+
+size_pattern = re.compile(r"\d+(b|m)")
+
+def get_model_size(model_name, model_info):
+    # In billions
+    try:
+        return model_info.safetensors["total"] / 1e9
+    except AttributeError:
+        #print(f"Repository {model_id} does not have safetensors weights")
+        pass
+    try:
+        size_match = re.search(size_pattern, model_name.lower())
+        size = size_match.group(0)
+        return int(size[:-1]) if size[-1] == "b" else int(size[:-1]) / 1e3
+    except AttributeError:
+        return None
+
+
+def apply_metadata(leaderboard_data: List[dict]):
+    get_model_infos_from_hub(leaderboard_data)

utils.py → src/auto_leaderboard/load_results.py

@@ -1,47 +1,23 @@
+from dataclasses import dataclass
+
 import glob
 import json
-from dataclasses import dataclass
 from typing import Dict, List, Tuple
 
+from src.utils_display import AutoEvalColumn, make_clickable_model
 import numpy as np
 
 # clone / pull the lmeh eval data
 METRICS = ["acc_norm", "acc_norm", "acc_norm", "mc2"]
 BENCHMARKS = ["arc_challenge", "hellaswag", "hendrycks", "truthfulqa_mc"]
 BENCH_TO_NAME = {
-    "arc_challenge": "ARC (25-shot) ⬆️",
-    "hellaswag": "HellaSwag (10-shot) ⬆️",
-    "hendrycks": "MMLU (5-shot) ⬆️",
-    "truthfulqa_mc": "TruthfulQA (0-shot) ⬆️",
+    "arc_challenge": AutoEvalColumn.arc.name,
+    "hellaswag": AutoEvalColumn.hellaswag.name,
+    "hendrycks": AutoEvalColumn.mmlu.name,
+    "truthfulqa_mc": AutoEvalColumn.truthfulqa.name,
 }
 
 
-def make_clickable_model(model_name):
-    LLAMAS = [
-        "huggingface/llama-7b",
-        "huggingface/llama-13b",
-        "huggingface/llama-30b",
-        "huggingface/llama-65b",
-    ]
-    if model_name in LLAMAS:
-        model = model_name.split("/")[1]
-        return f'<a target="_blank" href="https://ai.facebook.com/blog/large-language-model-llama-meta-ai/" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model}</a>'
-
-    if model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
-        link = "https://huggingface.co/" + "CarperAI/stable-vicuna-13b-delta"
-        return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">stable-vicuna-13b</a>'
-
-    if model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
-        link = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
-        return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">alpaca-13b</a>'
-
-    # remove user from model name
-    # model_name_show = ' '.join(model_name.split('/')[1:])
-
-    link = "https://huggingface.co/" + model_name
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
 @dataclass
 class EvalResult:
     eval_name: str
@@ -58,12 +34,12 @@ class EvalResult:
             base_model = f"{self.model}"
         data_dict = {}
 
-        data_dict["eval_name"] = self.eval_name
-        data_dict["8bit"] = self.is_8bit
-        data_dict["Model"] = make_clickable_model(base_model)
-        data_dict["model_name_for_query"] = base_model
-        data_dict["Revision"] = self.revision
-        data_dict["Average ⬆️"] = round(
+        data_dict["eval_name"] = self.eval_name # not a column, just a save name
+        data_dict[AutoEvalColumn.is_8bit.name] = self.is_8bit
+        data_dict[AutoEvalColumn.model.name] = make_clickable_model(base_model)
+        data_dict[AutoEvalColumn.dummy.name] = base_model
+        data_dict[AutoEvalColumn.revision.name] = self.revision
+        data_dict[AutoEvalColumn.average.name] = round(
             sum([v for k, v in self.results.items()]) / 4.0, 1
         )
 
@@ -88,17 +64,15 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, dict]:
     revision = path_split[-3]
     if len(path_split) == 7:
         # handles gpt2 type models that don't have an org
-        result_key = f"{path_split[-4]}_{path_split[-3]}_{path_split[-2]}"
+        result_key = f"{model}_{revision}_{is_8bit}"
     else:
-        result_key = (
-            f"{path_split[-5]}_{path_split[-4]}_{path_split[-3]}_{path_split[-2]}"
-        )
         org = path_split[-5]
+        result_key =  f"{org}_{model}_{revision}_{is_8bit}"
 
     eval_result = None
     for benchmark, metric in zip(BENCHMARKS, METRICS):
         if benchmark in json_filepath:
-            accs = np.array([v[metric] for k, v in data["results"].items()])
+            accs = np.array([v[metric] for v in data["results"].values()])
             mean_acc = round(np.mean(accs) * 100.0, 1)
             eval_result = EvalResult(
                 result_key, org, model, revision, is_8bit, {benchmark: mean_acc}
@@ -109,18 +83,19 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, dict]:
 
 def get_eval_results(is_public) -> List[EvalResult]:
     json_filepaths = glob.glob(
-        "evals/eval_results/public/**/16bit/*.json", recursive=True
+        "auto_evals/eval_results/public/**/16bit/*.json", recursive=True
     )
     if not is_public:
         json_filepaths += glob.glob(
-            "evals/eval_results/private/**/*.json", recursive=True
+            "auto_evals/eval_results/private/**/*.json", recursive=True
         )
         json_filepaths += glob.glob(
-            "evals/eval_results/private/**/*.json", recursive=True
+            "auto_evals/eval_results/private/**/*.json", recursive=True
         )
+        # include the 8bit evals of public models
         json_filepaths += glob.glob(
-            "evals/eval_results/public/**/8bit/*.json", recursive=True
-        )  # include the 8bit evals of public models
+            "auto_evals/eval_results/public/**/8bit/*.json", recursive=True
+        )  
     eval_results = {}
 
     for json_filepath in json_filepaths:
@@ -130,7 +105,7 @@ def get_eval_results(is_public) -> List[EvalResult]:
         else:
             eval_results[result_key] = eval_result
 
-    eval_results = [v for k, v in eval_results.items()]
+    eval_results = [v for v in eval_results.values()]
 
     return eval_results
 
@@ -139,12 +114,3 @@ def get_eval_results_dicts(is_public=True) -> List[Dict]:
     eval_results = get_eval_results(is_public)
 
     return [e.to_dict() for e in eval_results]
-
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

elo_utils.py → src/elo_leaderboard/load_results.py

@@ -6,9 +6,9 @@ import numpy as np
 import pandas as pd
 from datasets import load_dataset
 
-from content import PLOT_1_TITLE, PLOT_2_TITLE, PLOT_3_TITLE, PLOT_4_TITLE
-from utils import make_clickable_model
-from visualizations import (
+from src.assets.text_content import PLOT_1_TITLE, PLOT_2_TITLE, PLOT_3_TITLE, PLOT_4_TITLE
+from src.utils_display import make_clickable_model, EloEvalColumn
+from .visualizations import (
     get_bootstrap_result,
     switch_model_a_b,
     visualize_battle_count,
@@ -18,29 +18,6 @@ from visualizations import (
 )
 
 
-KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
-VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
-OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
-DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
-MODEL_PAGE = "https://huggingface.co/models"
-
-
-def make_clickable_model_elo(model_name):
-    link = ""
-    if model_name == "dolly-12b":
-        link = DOLLY_LINK
-    elif model_name == "vicuna-13b":
-        link = VICUNA_LINK
-    elif model_name == "koala-13b":
-        link = KOALA_LINK
-    elif model_name == "oasst-12b":
-        link = OASST_LINK
-    else:
-        link = MODEL_PAGE
-
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
 @dataclass
 class EloEvalResult:
     model: str
@@ -53,11 +30,11 @@ class EloEvalResult:
     def to_dict(self):
         base_model = f"{self.model}"
         data_dict = {}
-        data_dict["Model"] = make_clickable_model_elo(base_model)
-        data_dict["GPT-4 (all)"] = self.gpt_4_all
-        data_dict["Human (all)"] = self.human_all
-        data_dict["Human (instruct)"] = self.human_instruct
-        data_dict["Human (code-instruct)"] = self.human_code_instruct
+        data_dict[EloEvalColumn.model.name] = make_clickable_model(base_model)
+        data_dict[EloEvalColumn.gpt4.name] = self.gpt_4_all
+        data_dict[EloEvalColumn.human_all.name] = self.human_all
+        data_dict[EloEvalColumn.human_instruct.name] = self.human_instruct
+        data_dict[EloEvalColumn.human_code_instruct.name] = self.human_code_instruct
 
         return data_dict
 

visualizations.py → src/elo_leaderboard/visualizations.py

@@ -133,5 +133,5 @@ def visualize_rating_count(df, title):
     fig.update_layout(xaxis_title="model", yaxis_title="Rating Count", showlegend=False)
     fig.update_yaxes(range=[y_begin, y_end])
     # save the plot for the blog:
-    fig.write_html("model_counts.html", full_html=False, include_plotlyjs="cdn")
+    fig.write_html("src/assets/model_counts.html", full_html=False, include_plotlyjs="cdn")
     return fig

src/init.py

@@ -0,0 +1,73 @@
+import os 
+from huggingface_hub import Repository
+
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+
+
+def get_all_requested_models(requested_models_dir):
+    depth = 1
+    file_names = []
+
+    for root, dirs, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            file_names.extend([os.path.join(root, file) for file in files])
+
+    return set([file_name.lower().split("eval_requests/")[1] for file_name in file_names])
+
+def load_all_info_from_hub(LMEH_REPO, HUMAN_EVAL_REPO, GPT_4_EVAL_REPO):
+    auto_eval_repo = None
+    requested_models = None
+    if H4_TOKEN:
+        print("Pulling evaluation requests and results.")
+        # try:
+        #     shutil.rmtree("./auto_evals/")
+        # except:
+        #     pass
+
+        auto_eval_repo = Repository(
+            local_dir="./auto_evals/",
+            clone_from=LMEH_REPO,
+            use_auth_token=H4_TOKEN,
+            repo_type="dataset",
+        )
+        auto_eval_repo.git_pull()
+
+        requested_models_dir = "./auto_evals/eval_requests"
+        requested_models = get_all_requested_models(requested_models_dir)
+
+    human_eval_repo = None
+    if H4_TOKEN and not os.path.isdir("./human_evals"):
+        print("Pulling human evaluation repo")
+        human_eval_repo = Repository(
+            local_dir="./human_evals/",
+            clone_from=HUMAN_EVAL_REPO,
+            use_auth_token=H4_TOKEN,
+            repo_type="dataset",
+        )
+        human_eval_repo.git_pull()
+
+    gpt_4_eval_repo = None
+    if H4_TOKEN and not os.path.isdir("./gpt_4_evals"):
+        print("Pulling GPT-4 evaluation repo")
+        gpt_4_eval_repo = Repository(
+            local_dir="./gpt_4_evals/",
+            clone_from=GPT_4_EVAL_REPO,
+            use_auth_token=H4_TOKEN,
+            repo_type="dataset",
+        )
+        gpt_4_eval_repo.git_pull()
+
+    return auto_eval_repo, human_eval_repo, gpt_4_eval_repo, requested_models
+
+
+#def load_results(model, benchmark, metric):
+#    file_path = os.path.join("autoevals", model, f"{model}-eval_{benchmark}.json")
+#    if not os.path.exists(file_path):
+#        return 0.0, None
+
+#    with open(file_path) as fp:
+#        data = json.load(fp)
+#    accs = np.array([v[metric] for k, v in data["results"].items()])
+#    mean_acc = np.mean(accs)
+#    return mean_acc, data["config"]["model_args"]

src/utils_display.py

@@ -0,0 +1,96 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+    displayed_by_default: bool 
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model = ColumnContent("Model", "markdown", True)
+    revision = ColumnContent("Revision", "str", True)
+    is_8bit = ColumnContent("8bit", "bool", False)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    average = ColumnContent("Average ⬆️", "number", True)
+    arc = ColumnContent("ARC (25-s) ⬆️", "number", True)
+    hellaswag = ColumnContent("HellaSwag (10-s) ⬆️", "number", True)
+    mmlu = ColumnContent("MMLU (5-s) ⬆️", "number", True)
+    truthfulqa = ColumnContent("TruthfulQA (MC) (0-s) ⬆️", "number", True)
+    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
+
+@dataclass(frozen=True)
+class EloEvalColumn: # Elo evals column
+    model = ColumnContent("Model", "markdown", True)
+    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+    human_all = ColumnContent("Human (all)", "number", True)
+    human_instruct = ColumnContent("Human (instruct)", "number", True)
+    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn: # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    is_8bit = ColumnContent("8bit_eval", "bool", True)
+    has_delta_weight = ColumnContent("is_delta_weight", "bool", True)
+    status = ColumnContent("status", "str", True)
+
+LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]
+
+
+KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+MODEL_PAGE = "https://huggingface.co/models"
+LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+
+    if model_name in LLAMAS:
+        link = LLAMA_LINK
+        model_name = model_name.split("/")[1]
+    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+        link = VICUNA_LINK
+        model_name = "stable-vicuna-13b"
+    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+        link = ALPACA_LINK
+        model_name = "alpaca-13b"
+    if model_name == "dolly-12b":
+        link = DOLLY_LINK
+    elif model_name == "vicuna-13b":
+        link = VICUNA_LINK
+    elif model_name == "koala-13b":
+        link = KOALA_LINK
+    elif model_name == "oasst-12b":
+        link = OASST_LINK
+    #else:
+    #    link = MODEL_PAGE
+  
+    return model_hyperlink(link, model_name)
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"