[Add] git add

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -1,12 +1,13 @@
 ---
-title: LeaderboardTest
-emoji: ⚡
-colorFrom: gray
-colorTo: red
+title: Leaderboard Test
+emoji: 📉
+colorFrom: green
+colorTo: indigo
 sdk: gradio
-sdk_version: 3.43.2
+sdk_version: 3.27.0
 app_file: app.py
-pinned: false
+pinned: true
+license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,573 @@
+import json
+import os
+from datetime import datetime, timezone
+
+import gradio as gr
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import HfApi
+
+from src.assets.css_html_js import custom_css, get_window_url_params
+from src.assets.text_content import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    TITLE,
+)
+from src.display_models.get_model_metadata import DO_NOT_SUBMIT_MODELS, ModelType
+from src.display_models.utils import (
+    AutoEvalColumn,
+    EvalQueueColumn,
+    fields,
+    styled_error,
+    styled_message,
+    styled_warning,
+)
+from src.load_from_hub import get_evaluation_queue_df, get_leaderboard_df, is_model_on_hub, load_all_info_from_hub
+from src.rate_limiting import user_submission_permission
+
+pd.set_option("display.precision", 1)
+
+# clone / pull the lmeh eval data
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+
+QUEUE_REPO = "BearSean/leaderboard-test-requests"
+RESULTS_REPO = "BearSean/leaderboard-test-results"
+
+PRIVATE_QUEUE_REPO = "BearSean/leaderboard-test-requests"
+PRIVATE_RESULTS_REPO = "BearSean/leaderboard-test-results"
+
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+
+EVAL_REQUESTS_PATH = "eval-queue"
+EVAL_RESULTS_PATH = "eval-results"
+
+EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
+EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
+
+api = HfApi(token=H4_TOKEN)
+
+
+def restart_space():
+    api.restart_space(repo_id="BearSean/leaderboard-test", token=H4_TOKEN)
+
+# Rate limit variables
+RATE_LIMIT_PERIOD = 7
+RATE_LIMIT_QUOTA = 5
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+
+if not IS_PUBLIC:
+    COLS.insert(2, AutoEvalColumn.precision.name)
+    TYPES.insert(2, AutoEvalColumn.precision.type)
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [
+    c.name
+    for c in [
+        AutoEvalColumn.arc,
+        AutoEvalColumn.hellaswag,
+        AutoEvalColumn.mmlu,
+        AutoEvalColumn.truthfulqa,
+    ]
+]
+
+## LOAD INFO FROM HUB
+eval_queue, requested_models, eval_results, users_to_submission_dates = load_all_info_from_hub(
+    QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH
+)
+
+if not IS_PUBLIC:
+    (eval_queue_private, requested_models_private, eval_results_private, _) = load_all_info_from_hub(
+        PRIVATE_QUEUE_REPO,
+        PRIVATE_RESULTS_REPO,
+        EVAL_REQUESTS_PATH_PRIVATE,
+        EVAL_RESULTS_PATH_PRIVATE,
+    )
+else:
+    eval_queue_private, eval_results_private = None, None
+
+original_df = get_leaderboard_df(eval_results, eval_results_private, COLS, BENCHMARK_COLS)
+models = original_df["model_name_for_query"].tolist() # needed for model backlinks in their to the leaderboard
+
+# Commented out because it causes infinite restart loops in local
+# to_be_dumped = f"models = {repr(models)}\n"
+
+# with open("models_backlinks.py", "w") as f:
+#     f.write(to_be_dumped)
+
+# print(to_be_dumped)
+
+leaderboard_df = original_df.copy()
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df(eval_queue, eval_queue_private, EVAL_REQUESTS_PATH, EVAL_COLS)
+
+
+## INTERACTION FUNCTIONS
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    private: bool,
+    weight_type: str,
+    model_type: str,
+):
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    num_models_submitted_in_period = user_submission_permission(model, users_to_submission_dates, RATE_LIMIT_PERIOD)
+    if num_models_submitted_in_period > RATE_LIMIT_QUOTA:
+        error_msg = f"Organisation or user `{model.split('/')[0]}`"
+        error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
+        error_msg += f"in the last {RATE_LIMIT_PERIOD} days.\n"
+        error_msg += "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
+        return styled_error(error_msg)
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # check the model actually exists before adding the eval
+    if revision == "":
+        revision = "main"
+
+    if weight_type in ["Delta", "Adapter"]:
+        base_model_on_hub, error = is_model_on_hub(base_model, revision)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    if not weight_type == "Adapter":
+        model_on_hub, error = is_model_on_hub(model, revision)
+        if not model_on_hub:
+            return styled_error(f'Model "{model}" {error}')
+
+    print("adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "private": private,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type,
+    }
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
+
+    # Check if the model has been forbidden:
+    if out_path.split("eval-queue/")[1] in DO_NOT_SUBMIT_MODELS:
+        return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
+
+    # Check for duplicate submission
+    if f"{model}_{revision}_{precision}" in requested_models:
+        return styled_warning("This model has been already submitted.")
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    api.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )
+
+
+# Basics
+def refresh() -> list[pd.DataFrame]:
+    leaderboard_df = get_leaderboard_df(eval_results, eval_results_private, COLS, BENCHMARK_COLS)
+    (
+        finished_eval_queue_df,
+        running_eval_queue_df,
+        pending_eval_queue_df,
+    ) = get_evaluation_queue_df(eval_queue, eval_queue_private, EVAL_REQUESTS_PATH, EVAL_COLS)
+    return (
+        leaderboard_df,
+        finished_eval_queue_df,
+        running_eval_queue_df,
+        pending_eval_queue_df,
+    )
+
+
+def change_tab(query_param: str):
+    query_param = query_param.replace("'", '"')
+    query_param = json.loads(query_param)
+
+    if isinstance(query_param, dict) and "tab" in query_param and query_param["tab"] == "evaluation":
+        return gr.Tabs.update(selected=1)
+    else:
+        return gr.Tabs.update(selected=0)
+
+
+# Searching and filtering
+def search_table(df: pd.DataFrame, current_columns_df: pd.DataFrame, query: str) -> pd.DataFrame:
+    current_columns = current_columns_df.columns
+    if AutoEvalColumn.model_type.name in current_columns:
+        filtered_df = df[
+            (df[AutoEvalColumn.dummy.name].str.contains(query, case=False))
+            | (df[AutoEvalColumn.model_type.name].str.contains(query, case=False))
+        ]
+    else:
+        filtered_df = df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
+    return filtered_df[current_columns]
+
+
+def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+    always_here_cols = [
+        AutoEvalColumn.model_type_symbol.name,
+        AutoEvalColumn.model.name,
+    ]
+    # We use COLS to maintain sorting
+    filtered_df = df[
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]
+    ]
+    return filtered_df
+
+NUMERIC_INTERVALS = {
+    "< 1.5B": (0, 1.5),
+    "~3B": (1.5, 5),
+    "~7B": (6, 11),
+    "~13B": (12, 15),
+    # "~35B": (16, 55),
+    # "60B+": (55, 10000),
+}
+
+def filter_models(
+    df: pd.DataFrame, current_columns_df: pd.DataFrame, type_query: list, size_query: list, show_deleted: bool
+) -> pd.DataFrame:
+    current_columns = current_columns_df.columns
+
+    # Show all models
+    if show_deleted:
+        filtered_df = df[current_columns]
+    else:  # Show only still on the hub models
+        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True][current_columns]
+
+    type_emoji = [t[0] for t in type_query]
+    filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+
+    numeric_interval = [NUMERIC_INTERVALS[s] for s in size_query]
+    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+    filtered_df = filtered_df[params_column.between(numeric_interval[0][0], numeric_interval[-1][1])]
+
+    return filtered_df
+
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[
+                                c
+                                for c in COLS
+                                if c
+                                not in [
+                                    AutoEvalColumn.dummy.name,
+                                    AutoEvalColumn.model.name,
+                                    AutoEvalColumn.model_type_symbol.name,
+                                    AutoEvalColumn.still_on_hub.name,
+                                ]
+                            ],
+                            value=[
+                                c
+                                for c in COLS_LITE
+                                if c
+                                not in [
+                                    AutoEvalColumn.dummy.name,
+                                    AutoEvalColumn.model.name,
+                                    AutoEvalColumn.model_type_symbol.name,
+                                    AutoEvalColumn.still_on_hub.name,
+                                ]
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    with gr.Row():
+                        deleted_models_visibility = gr.Checkbox(
+                            value=True, label="👀 삭제/비공개된 모델도 함께 보기", interactive=True
+                        )
+                with gr.Column(min_width=320):
+                    search_bar = gr.Textbox(
+                        placeholder="🔍 찾고자 하는 모델 명을 입력하세요",
+                        show_label=False,
+                        elem_id="search-bar",
+                    )
+                    with gr.Box(elem_id="box-filter"):
+                        filter_columns_type = gr.CheckboxGroup(
+                            label="Model types",
+                            choices=[
+                                ModelType.PT.to_str(),
+                                ModelType.FT.to_str(),
+                                ModelType.IFT.to_str(),
+                                ModelType.RL.to_str(),
+                            ],
+                            value=[
+                                ModelType.PT.to_str(),
+                                ModelType.FT.to_str(),
+                                ModelType.IFT.to_str(),
+                                ModelType.RL.to_str(),
+                            ],
+                            interactive=True,
+                            elem_id="filter-columns-type",
+                        )
+                        filter_columns_size = gr.CheckboxGroup(
+                            label="Model sizes",
+                            choices=list(NUMERIC_INTERVALS.keys()),
+                            value=list(NUMERIC_INTERVALS.keys()),
+                            interactive=True,
+                            elem_id="filter-columns-size",
+                        )
+
+            leaderboard_table = gr.components.Dataframe(
+                value=leaderboard_df[
+                    [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name]
+                    + shown_columns.value
+                    + [AutoEvalColumn.dummy.name]
+                ],
+                headers=[
+                    AutoEvalColumn.model_type_symbol.name,
+                    AutoEvalColumn.model.name,
+                ]
+                + shown_columns.value
+                + [AutoEvalColumn.dummy.name],
+                datatype=TYPES,
+                max_rows=None,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=original_df,
+                headers=COLS,
+                datatype=TYPES,
+                max_rows=None,
+                visible=False,
+            )
+            search_bar.submit(
+                search_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    search_bar,
+                ],
+                leaderboard_table,
+            )
+            shown_columns.change(
+                select_columns,
+                [hidden_leaderboard_table_for_search, shown_columns],
+                leaderboard_table,
+                queue=False,
+            )
+            filter_columns_type.change(
+                filter_models,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    filter_columns_type,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                ],
+                leaderboard_table,
+                queue=False,
+            )
+            filter_columns_size.change(
+                filter_models,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    filter_columns_type,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                ],
+                leaderboard_table,
+                queue=False,
+            )
+            deleted_models_visibility.change(
+                filter_models,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    filter_columns_type,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                ],
+                leaderboard_table,
+                queue=False,
+            )
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+            with gr.Column():
+                with gr.Row():
+                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+                with gr.Column():
+                    with gr.Accordion(
+                        f"✅ 평가 완료 ({len(finished_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            finished_eval_table = gr.components.Dataframe(
+                                value=finished_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                max_rows=5,
+                            )
+                    with gr.Accordion(
+                        f"🔄 평가 진행 중 ({len(running_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            running_eval_table = gr.components.Dataframe(
+                                value=running_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                max_rows=5,
+                            )
+
+                    with gr.Accordion(
+                        f"⏳ 평가 대기 중 ({len(pending_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            pending_eval_table = gr.components.Dataframe(
+                                value=pending_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                max_rows=5,
+                            )
+            with gr.Row():
+                gr.Markdown("# ✉️✨ 여기에서 모델을 제출해주세요!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name")
+                    revision_name_textbox = gr.Textbox(label="revision", placeholder="main")
+                    private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
+                    model_type = gr.Dropdown(
+                        choices=[
+                            ModelType.PT.to_str(" : "),
+                            ModelType.FT.to_str(" : "),
+                            ModelType.IFT.to_str(" : "),
+                            ModelType.RL.to_str(" : "),
+                        ],
+                        label="Model type",
+                        multiselect=False,
+                        value=None,
+                        interactive=True,
+                    )
+
+                with gr.Column():
+                    precision = gr.Dropdown(
+                        choices=[
+                            "float16",
+                            "bfloat16",
+                            "8bit (LLM.int8)",
+                            "4bit (QLoRA / FP4)",
+                            "GPTQ"
+                        ],
+                        label="Precision",
+                        multiselect=False,
+                        value="float16",
+                        interactive=True,
+                    )
+                    weight_type = gr.Dropdown(
+                        choices=["Original", "Delta", "Adapter"],
+                        label="Weights type",
+                        multiselect=False,
+                        value="Original",
+                        interactive=True,
+                    )
+                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+
+            submit_button = gr.Button("제출하고 평가받기")
+            submission_result = gr.Markdown()
+            submit_button.click(
+                add_new_eval,
+                [
+                    model_name_textbox,
+                    base_model_name_textbox,
+                    revision_name_textbox,
+                    precision,
+                    private,
+                    weight_type,
+                    model_type,
+                ],
+                submission_result,
+            )
+
+        with gr.Row():
+            refresh_button = gr.Button("새로고침")
+            refresh_button.click(
+                refresh,
+                inputs=[],
+                outputs=[
+                    leaderboard_table,
+                    finished_eval_table,
+                    running_eval_table,
+                    pending_eval_table,
+                ],
+                api_name='refresh'
+            )
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                elem_id="citation-button",
+            ).style(show_copy_button=True)
+
+    dummy = gr.Textbox(visible=False)
+    demo.load(
+        change_tab,
+        dummy,
+        tabs,
+        _js=get_window_url_params,
+    )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=3600)
+scheduler.start()
+demo.queue(concurrency_count=40).launch()

models_backlinks.py

@@ -0,0 +1 @@
+models = ['upstage/Llama-2-70b-instruct-v2', 'upstage/Llama-2-70b-instruct', 'upstage/llama-65b-instruct', 'upstage/llama-65b-instruct', 'upstage/llama-30b-instruct-2048', 'upstage/llama-30b-instruct', 'baseline']

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,70 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+APScheduler==3.10.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+datasets==2.12.0
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.11.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+gradio==3.27.0
+gradio_client==0.1.3
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.16.4
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.0
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.2
+orjson==3.8.10
+packaging==23.1
+pandas==2.0.0
+Pillow==9.5.0
+plotly==5.14.1
+pyarrow==11.0.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+requests==2.28.2
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+transformers==4.32.0
+typing_extensions==4.5.0
+tzdata==2023.3
+tzlocal==4.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+yarl==1.8.2

src/assets/css_html_js.py

@@ -0,0 +1,111 @@
+custom_css = """
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+#leaderboard-table {
+    margin-top: 15px
+}
+
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+}
+
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+    display: none;
+}
+
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+
+#scale-logo .download {
+    display: none;
+}
+#filter_type{
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span{
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap{
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner{  
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input{
+    width: 1px
+}
+#filter-columns-type{
+    border:0;
+    padding:0.5;
+}
+#filter-columns-size{
+    border:0;
+    padding:0.5;
+}
+#box-filter > .form{
+    border: 0
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/assets/hardcoded_evals.py

@@ -0,0 +1,40 @@
+from src.display_models.utils import AutoEvalColumn, model_hyperlink
+
+gpt4_values = {
+    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt4"),
+    AutoEvalColumn.revision.name: "tech report",
+    AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.average.name: 84.3,
+    AutoEvalColumn.arc.name: 96.3,
+    AutoEvalColumn.hellaswag.name: 95.3,
+    AutoEvalColumn.mmlu.name: 86.4,
+    AutoEvalColumn.truthfulqa.name: 59.0,
+    AutoEvalColumn.dummy.name: "GPT-4",
+    AutoEvalColumn.model_type.name: "",
+}
+
+gpt35_values = {
+    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt3.5"),
+    AutoEvalColumn.revision.name: "tech report",
+    AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.average.name: 71.9,
+    AutoEvalColumn.arc.name: 85.2,
+    AutoEvalColumn.hellaswag.name: 85.5,
+    AutoEvalColumn.mmlu.name: 70.0,
+    AutoEvalColumn.truthfulqa.name: 47.0,
+    AutoEvalColumn.dummy.name: "GPT-3.5",
+    AutoEvalColumn.model_type.name: "",
+}
+
+baseline = {
+    AutoEvalColumn.model.name: "<p>Baseline</p>",
+    AutoEvalColumn.revision.name: "N/A",
+    AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.average.name: 25.0,
+    AutoEvalColumn.arc.name: 25.0,
+    AutoEvalColumn.hellaswag.name: 25.0,
+    AutoEvalColumn.mmlu.name: 25.0,
+    AutoEvalColumn.truthfulqa.name: 25.0,
+    AutoEvalColumn.dummy.name: "baseline",
+    AutoEvalColumn.model_type.name: "",
+}

src/assets/text_content.py

@@ -0,0 +1,96 @@
+from src.display_models.model_metadata_type import ModelType
+
+TITLE = """<h1 align="center" id="space-title">🚀 Open Ko-LLM Leaderboard</h1>"""
+
+INTRODUCTION_TEXT = f"""
+🚀 Open Ko-LLM Leaderboard는 한국어 초거대 언어모델의 성능을 객관적으로 평가합니다.
+
+"제출" 페이지에서 모델 제출 시 자동으로 평가됩니다. 평가에 사용되는 GPU는 KT의 지원으로 운영됩니다.
+평가에 사용되는 데이터는 전문 지식, 추론 능력, 환각, 윤리, 상식의 다섯가지 요소를 평가하기 위한 데이터셋으로 구성되어 있습니다.
+벤치마크 데이터셋에 대한 더 자세한 정보는 "정보" 페이지에서 제공되고 있습니다.
+
+업스테이지와 NIA가 공동 주최하며 업스테이지가 운영합니다.
+"""
+
+LLM_BENCHMARKS_TEXT = f"""
+# Context
+뛰어난 LLM 모델들이 앞다투어 공개되고 있지만 이는 대부분 영어 중심의, 영어 문화권에 익숙한 모델입니다. 저희는 한국어 리더보드 🚀 Open Ko-LLM을 운영하여 한국어와 한국 문화의 특성을 반영한 모델을 평가하고자 합니다. 이를 통해 한국어 사용자들이 편리하게 리더보드를 이용하고 참여하여 한국의 연구 수준 향상에 기여할 수 있기를 바랍니다. 
+
+## Icons
+{ModelType.PT.to_str(" : ")} model
+{ModelType.FT.to_str(" : ")} model
+{ModelType.IFT.to_str(" : ")} model
+{ModelType.RL.to_str(" : ")} model
+만약 아이콘이 없다면 아직 모델에 대한 정보가 부족함을 나타냅니다.
+모델에 대한 정보는 issue를 통해 전달해주세요! 🤩
+
+🏴‍☠️ : 해당 아이콘은 이 모델이 커뮤니티에 의해 주의 대상으로 선정되었으므로 이용 자제를 바란다는 의미입니다. 아이콘을 클릭 시 해당 모델에 대한 discussion으로 이동합니다.
+(높은 리더보드 순위를 위해 평가셋을 학습에 이용한 모델 등이 주의 대상으로 선정됩니다)
+
+## How it works
+
+📈 HuggingFace OpenLLM에서 운영하는 4개의 태스크(HellaSwag, MMLU, Arc, Truthful QA)의 데이터를 한국어로 번역한 데이터셋을 비롯해 총 6가지의 데이터로 벤치마크를 구성했습니다. 
+- Ko-HellaSwag (업스테이지 제공)
+- Ko-MMLU (업스테이지 제공)
+- Ko-Arc (업스테이지 제공)
+- Ko-Truthful QA (업스테이지 제공)
+- KoCommongen (NIA 한국지능정보사회진흥원 제공)
+- 텍스트 윤리검증 데이터 (NIA 한국지능정보사회진흥원 제공)
+LLM 시대에 걸맞는 평가를 위해 상식, 전문 지식, 추론, 환각, 윤리의 다섯가지 요소를 평가하기에 적합한 데이터셋들을 벤치마크로 선정했습니다. 최종 점수는 6개의 평가 데이터에 대한 평균 점수로 환산합니다.
+
+KT로부터 평가에 사용되는 GPU를 제공받았습니다.
+
+## 좀 더 자세한 정보
+- 좀 더 자세한 수치 정보는: https://huggingface.co/datasets/open-llm-leaderboard/results
+- 모델의 입출력에 대한 자세한 정보는: https://huggingface.co/datasets/open-llm-leaderboard/details
+- 모델의 평가 큐와 평가 상태는: https://huggingface.co/datasets/open-llm-leaderboard/requests
+
+## 결과 재현
+평가 결과를 재현하기 위해서는 [이 버전](https://github.com/EleutherAI/lm-evaluation-harness/tree/b281b0921b636bc36ad05c0b0b0763bd6dd43463)의 데이터셋을 이용하세요. (밑에는 코드 및 평가 환경이라서 일단 skip)
+
+## 더보기
+질문이 있으시면 [여기](https://huggingface.co/spaces/BearSean/leaderboard-test/discussions/1)서 FAQ를 확인하실 수 있습니다! 
+또한 커뮤니티, 다른 팀 및 연구소에서 제공하는 멋진 자료를 [여기](https://huggingface.co/spaces/BearSean/leaderboard-test/discussions/2)에 모아 두었습니다. 추가로 제안 사항이 있으시다면 이 곳을 이용해주세요.
+"""
+
+EVALUATION_QUEUE_TEXT = f"""
+# 🚀 Open-Ko LLM 리더보드의 평가 큐입니다.
+이곳에 추가된 모델들은 곧 자동적으로 KT의 GPU 위에서 평가될 예정입니다!
+
+## <모델 제출 전 확인하면 좋은 것들>
+
+### 1️⃣ 모델과 토크나이저가 AutoClasses로 불러올 수 있나요?
+```
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+
+만약 이 단계가 실패했다면 에러 메세지를 따라 모델을 디버깅한 후에 제출해주세요. 
+⚠️ 모델이 public 상태여야 합니다!
+⚠️ 만약 모델이 use_remote_code=True여야 한다면 잠시 기다려주세요. 현재로서는 아직 이 옵션을 지원하지 않지만 작동할 수 있도록 하고 있습니다!
+
+### 2️⃣ 모델의 weight를 safetensors로 바꿨나요?
+safetensors는 weight를 보관하는 새로운 포맷으로, 훨씬 안전하고 빠르게 사용할 수 있습니다. 또한 모델의 parameter 개수를 Extended Viewer에 추가할 수 있습니다
+
+### 3️⃣ 모델이 오픈 라이센스를 따르나요?
+🚀 Open-Ko LLM은 Open LLM을 위한 리더보드로, 많은 사람들이 다양한 모델을 사용하기를 바랍니다
+
+### 4️⃣ 모델 카드를 작성하셨나요?
+리더보드에 모델에 대한 추가 정보를 업로드할 때 작성하신 모델 카드가 업로드됩니다
+
+## 모델이 실패한 경우:
+만약 제출한 모델의 상태가 FAILED가 된다면 이는 모델이 실행 중단되었음을 의미합니다. 먼저 위의 네 단계를 모두 따랐는지 확인해보세요. 모든 단계를 따랐음에도 불구하고 실행 중단되었을 때는 EleutherAIHarness 를 로컬에서 실행할 수 있는지 확인하기 위해 위의 코드를 수정 없이 실행하세요. (태스크 별 예시의 수를 제한하기 위해 —limit 파라미터를 추가할 수 있습니다.)
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""
+@misc{open-ko-llm-leaderboard,
+  author = {Chanjun Park, Hwalsuk Lee, Hyunbyung Park, Sanghoon Kim, Seonghwan Cho, Sunghun Kim, Sukyung Lee},
+  title = {Open Ko LLM Leaderboard},
+  year = {2023},
+  publisher = {Upstage, National Information Society Agency},
+  howpublished = "\url{https://huggingface.co/spaces/BearSean/leaderboard-test/discussions/1}"
+}
+"""

src/display_models/get_model_metadata.py

@@ -0,0 +1,132 @@
+import glob
+import json
+import os
+import re
+from typing import List
+
+import huggingface_hub
+from huggingface_hub import HfApi
+from tqdm import tqdm
+
+from src.display_models.model_metadata_flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
+from src.display_models.model_metadata_type import MODEL_TYPE_METADATA, ModelType, model_type_from_str
+from src.display_models.utils import AutoEvalColumn, model_hyperlink
+
+api = HfApi(token=os.environ.get("H4_TOKEN", None))
+
+
+def get_model_infos_from_hub(leaderboard_data: List[dict]):
+    for model_data in tqdm(leaderboard_data):
+        model_name = model_data["model_name_for_query"]
+        try:
+            model_info = api.model_info(model_name)
+        except huggingface_hub.utils._errors.RepositoryNotFoundError:
+            print("Repo not found!", model_name)
+            model_data[AutoEvalColumn.license.name] = None
+            model_data[AutoEvalColumn.likes.name] = None
+            model_data[AutoEvalColumn.params.name] = get_model_size(model_name, None)
+            continue
+
+        model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
+        model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
+        model_data[AutoEvalColumn.params.name] = get_model_size(model_name, model_info)
+
+
+def get_model_license(model_info):
+    try:
+        return model_info.cardData["license"]
+    except Exception:
+        return None
+
+
+def get_model_likes(model_info):
+    return model_info.likes
+
+
+size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
+
+
+def get_model_size(model_name, model_info):
+    # In billions
+    try:
+        return round(model_info.safetensors["total"] / 1e9, 3)
+    except AttributeError:
+        try:
+            size_match = re.search(size_pattern, model_name.lower())
+            size = size_match.group(0)
+            return round(float(size[:-1]) if size[-1] == "b" else float(size[:-1]) / 1e3, 3)
+        except AttributeError:
+            return None
+
+
+def get_model_type(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        request_files = os.path.join(
+            "eval-queue",
+            model_data["model_name_for_query"] + "_eval_request_*" + ".json",
+        )
+        request_files = glob.glob(request_files)
+
+        # Select correct request file (precision)
+        request_file = ""
+        if len(request_files) == 1:
+            request_file = request_files[0]
+        elif len(request_files) > 1:
+            request_files = sorted(request_files, reverse=True)
+            for tmp_request_file in request_files:
+                with open(tmp_request_file, "r") as f:
+                    req_content = json.load(f)
+                    if (
+                        req_content["status"] == "FINISHED"
+                        and req_content["precision"] == model_data["Precision"].split(".")[-1]
+                    ):
+                        request_file = tmp_request_file
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            model_type = model_type_from_str(request["model_type"])
+            model_data[AutoEvalColumn.model_type.name] = model_type.value.name
+            model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol  # + ("🔺" if is_delta else "")
+        except Exception:
+            if model_data["model_name_for_query"] in MODEL_TYPE_METADATA:
+                model_data[AutoEvalColumn.model_type.name] = MODEL_TYPE_METADATA[
+                    model_data["model_name_for_query"]
+                ].value.name
+                model_data[AutoEvalColumn.model_type_symbol.name] = MODEL_TYPE_METADATA[
+                    model_data["model_name_for_query"]
+                ].value.symbol  # + ("🔺" if is_delta else "")
+            else:
+                model_data[AutoEvalColumn.model_type.name] = ModelType.Unknown.value.name
+                model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol
+
+
+def flag_models(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        if model_data["model_name_for_query"] in FLAGGED_MODELS:
+            issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
+            issue_link = model_hyperlink(
+                FLAGGED_MODELS[model_data["model_name_for_query"]],
+                f"See discussion #{issue_num}",
+            )
+            model_data[
+                AutoEvalColumn.model.name
+            ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
+
+
+def remove_forbidden_models(leaderboard_data: List[dict]):
+    indices_to_remove = []
+    for ix, model in enumerate(leaderboard_data):
+        if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
+            indices_to_remove.append(ix)
+
+    for ix in reversed(indices_to_remove):
+        leaderboard_data.pop(ix)
+    return leaderboard_data
+
+
+def apply_metadata(leaderboard_data: List[dict]):
+    leaderboard_data = remove_forbidden_models(leaderboard_data)
+    get_model_type(leaderboard_data)
+    get_model_infos_from_hub(leaderboard_data)
+    flag_models(leaderboard_data)

src/display_models/model_metadata_flags.py

@@ -0,0 +1,15 @@
+# Models which have been flagged by users as being problematic for a reason or another
+# (Model name to forum discussion link)
+FLAGGED_MODELS = {
+    "Voicelab/trurl-2-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/202",
+    "deepnight-research/llama-2-70B-inst": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/207",
+    "Aspik101/trurl-2-13b-pl-instruct_unload": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/213",
+    "Fredithefish/ReasonixPajama-3B-HF": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/236",
+    "TigerResearch/tigerbot-7b-sft-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/237",
+    "gaodrew/gaodrew-gorgonzola-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/215",
+}
+
+# Models which have been requested by orgs to not be submitted on the leaderboard
+DO_NOT_SUBMIT_MODELS = [
+    "Voicelab/trurl-2-13b",  # trained on MMLU
+]

src/display_models/model_metadata_type.py

@@ -0,0 +1,553 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict
+
+
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str  # emoji
+
+
+class ModelType(Enum):
+    PT = ModelInfo(name="pretrained", symbol="🟢")
+    FT = ModelInfo(name="fine-tuned", symbol="🔶")
+    IFT = ModelInfo(name="instruction-tuned", symbol="⭕")
+    RL = ModelInfo(name="RL-tuned", symbol="🟦")
+    Unknown = ModelInfo(name="Unknown, add type to request file!", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+
+MODEL_TYPE_METADATA: Dict[str, ModelType] = {
+    "tiiuae/falcon-180B": ModelType.PT,
+    "Qwen/Qwen-7B": ModelType.PT,
+    "Qwen/Qwen-7B-Chat": ModelType.RL,
+    "notstoic/PygmalionCoT-7b": ModelType.IFT,
+    "aisquared/dlite-v1-355m": ModelType.IFT,
+    "aisquared/dlite-v1-1_5b": ModelType.IFT,
+    "aisquared/dlite-v1-774m": ModelType.IFT,
+    "aisquared/dlite-v1-124m": ModelType.IFT,
+    "aisquared/chopt-2_7b": ModelType.IFT,
+    "aisquared/dlite-v2-124m": ModelType.IFT,
+    "aisquared/dlite-v2-774m": ModelType.IFT,
+    "aisquared/dlite-v2-1_5b": ModelType.IFT,
+    "aisquared/chopt-1_3b": ModelType.IFT,
+    "aisquared/dlite-v2-355m": ModelType.IFT,
+    "augtoma/qCammel-13": ModelType.IFT,
+    "Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload": ModelType.IFT,
+    "Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload": ModelType.IFT,
+    "TheBloke/alpaca-lora-65B-HF": ModelType.FT,
+    "TheBloke/tulu-7B-fp16": ModelType.IFT,
+    "TheBloke/guanaco-7B-HF": ModelType.FT,
+    "TheBloke/koala-7B-HF": ModelType.FT,
+    "TheBloke/wizardLM-7B-HF": ModelType.IFT,
+    "TheBloke/airoboros-13B-HF": ModelType.IFT,
+    "TheBloke/koala-13B-HF": ModelType.FT,
+    "TheBloke/Wizard-Vicuna-7B-Uncensored-HF": ModelType.FT,
+    "TheBloke/dromedary-65b-lora-HF": ModelType.IFT,
+    "TheBloke/wizardLM-13B-1.0-fp16": ModelType.IFT,
+    "TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16": ModelType.FT,
+    "TheBloke/Wizard-Vicuna-30B-Uncensored-fp16": ModelType.FT,
+    "TheBloke/wizard-vicuna-13B-HF": ModelType.IFT,
+    "TheBloke/UltraLM-13B-fp16": ModelType.IFT,
+    "TheBloke/OpenAssistant-FT-7-Llama-30B-HF": ModelType.FT,
+    "TheBloke/vicuna-13B-1.1-HF": ModelType.IFT,
+    "TheBloke/guanaco-13B-HF": ModelType.FT,
+    "TheBloke/guanaco-65B-HF": ModelType.FT,
+    "TheBloke/airoboros-7b-gpt4-fp16": ModelType.IFT,
+    "TheBloke/llama-30b-supercot-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/Llama-2-13B-fp16": ModelType.PT,
+    "TheBloke/llama-2-70b-Guanaco-QLoRA-fp16": ModelType.FT,
+    "TheBloke/landmark-attention-llama7b-fp16": ModelType.IFT,
+    "TheBloke/Planner-7B-fp16": ModelType.IFT,
+    "TheBloke/Wizard-Vicuna-13B-Uncensored-HF": ModelType.FT,
+    "TheBloke/gpt4-alpaca-lora-13B-HF": ModelType.IFT,
+    "TheBloke/gpt4-x-vicuna-13B-HF": ModelType.IFT,
+    "TheBloke/gpt4-alpaca-lora_mlp-65B-HF": ModelType.IFT,
+    "TheBloke/tulu-13B-fp16": ModelType.IFT,
+    "TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16": ModelType.IFT,
+    "TheBloke/Llama-2-70B-fp16": ModelType.IFT,
+    "TheBloke/WizardLM-30B-fp16": ModelType.IFT,
+    "TheBloke/robin-13B-v2-fp16": ModelType.FT,
+    "TheBloke/robin-33B-v2-fp16": ModelType.FT,
+    "TheBloke/Vicuna-13B-CoT-fp16": ModelType.IFT,
+    "TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16": ModelType.FT,
+    "TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/GPlatty-30B-SuperHOT-8K-fp16": ModelType.FT,
+    "TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16": ModelType.IFT,
+    "jphme/orca_mini_v2_ger_7b": ModelType.IFT,
+    "Ejafa/vicuna_7B_vanilla_1.1": ModelType.FT,
+    "kevinpro/Vicuna-13B-CoT": ModelType.IFT,
+    "AlekseyKorshuk/pygmalion-6b-vicuna-chatml": ModelType.FT,
+    "AlekseyKorshuk/chatml-pyg-v1": ModelType.FT,
+    "concedo/Vicuzard-30B-Uncensored": ModelType.FT,
+    "concedo/OPT-19M-ChatSalad": ModelType.FT,
+    "concedo/Pythia-70M-ChatSalad": ModelType.FT,
+    "digitous/13B-HyperMantis": ModelType.IFT,
+    "digitous/Adventien-GPTJ": ModelType.FT,
+    "digitous/Alpacino13b": ModelType.IFT,
+    "digitous/GPT-R": ModelType.IFT,
+    "digitous/Javelin-R": ModelType.IFT,
+    "digitous/Javalion-GPTJ": ModelType.IFT,
+    "digitous/Javalion-R": ModelType.IFT,
+    "digitous/Skegma-GPTJ": ModelType.FT,
+    "digitous/Alpacino30b": ModelType.IFT,
+    "digitous/Janin-GPTJ": ModelType.FT,
+    "digitous/Janin-R": ModelType.FT,
+    "digitous/Javelin-GPTJ": ModelType.FT,
+    "SaylorTwift/gpt2_test": ModelType.PT,
+    "anton-l/gpt-j-tiny-random": ModelType.FT,
+    "Andron00e/YetAnother_Open-Llama-3B-LoRA-OpenOrca": ModelType.FT,
+    "Lazycuber/pyg-instruct-wizardlm": ModelType.FT,
+    "Lazycuber/Janemalion-6B": ModelType.FT,
+    "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1": ModelType.FT,
+    "IDEA-CCNL/Ziya-LLaMA-13B-v1": ModelType.IFT,
+    "dsvv-cair/alpaca-cleaned-llama-30b-bf16": ModelType.FT,
+    "gpt2-medium": ModelType.PT,
+    "camel-ai/CAMEL-13B-Combined-Data": ModelType.IFT,
+    "camel-ai/CAMEL-13B-Role-Playing-Data": ModelType.FT,
+    "camel-ai/CAMEL-33B-Combined-Data": ModelType.IFT,
+    "PygmalionAI/pygmalion-6b": ModelType.FT,
+    "PygmalionAI/metharme-1.3b": ModelType.IFT,
+    "PygmalionAI/pygmalion-1.3b": ModelType.FT,
+    "PygmalionAI/pygmalion-350m": ModelType.FT,
+    "PygmalionAI/pygmalion-2.7b": ModelType.FT,
+    "medalpaca/medalpaca-7b": ModelType.FT,
+    "lilloukas/Platypus-30B": ModelType.IFT,
+    "lilloukas/GPlatty-30B": ModelType.FT,
+    "mncai/chatdoctor": ModelType.FT,
+    "chaoyi-wu/MedLLaMA_13B": ModelType.FT,
+    "LoupGarou/WizardCoder-Guanaco-15B-V1.0": ModelType.IFT,
+    "LoupGarou/WizardCoder-Guanaco-15B-V1.1": ModelType.FT,
+    "hakurei/instruct-12b": ModelType.IFT,
+    "hakurei/lotus-12B": ModelType.FT,
+    "shibing624/chinese-llama-plus-13b-hf": ModelType.IFT,
+    "shibing624/chinese-alpaca-plus-7b-hf": ModelType.IFT,
+    "shibing624/chinese-alpaca-plus-13b-hf": ModelType.IFT,
+    "mosaicml/mpt-7b-instruct": ModelType.IFT,
+    "mosaicml/mpt-30b-chat": ModelType.IFT,
+    "mosaicml/mpt-7b-storywriter": ModelType.FT,
+    "mosaicml/mpt-30b-instruct": ModelType.IFT,
+    "mosaicml/mpt-7b-chat": ModelType.IFT,
+    "mosaicml/mpt-30b": ModelType.PT,
+    "Corianas/111m": ModelType.IFT,
+    "Corianas/Quokka_1.3b": ModelType.IFT,
+    "Corianas/256_5epoch": ModelType.FT,
+    "Corianas/Quokka_256m": ModelType.IFT,
+    "Corianas/Quokka_590m": ModelType.IFT,
+    "Corianas/gpt-j-6B-Dolly": ModelType.FT,
+    "Corianas/Quokka_2.7b": ModelType.IFT,
+    "cyberagent/open-calm-7b": ModelType.FT,
+    "Aspik101/Nous-Hermes-13b-pl-lora_unload": ModelType.IFT,
+    "THUDM/chatglm2-6b": ModelType.IFT,
+    "MetaIX/GPT4-X-Alpasta-30b": ModelType.IFT,
+    "NYTK/PULI-GPTrio": ModelType.PT,
+    "EleutherAI/pythia-1.3b": ModelType.PT,
+    "EleutherAI/pythia-2.8b-deduped": ModelType.PT,
+    "EleutherAI/gpt-neo-125m": ModelType.PT,
+    "EleutherAI/pythia-160m": ModelType.PT,
+    "EleutherAI/gpt-neo-2.7B": ModelType.PT,
+    "EleutherAI/pythia-1b-deduped": ModelType.PT,
+    "EleutherAI/pythia-6.7b": ModelType.PT,
+    "EleutherAI/pythia-70m-deduped": ModelType.PT,
+    "EleutherAI/gpt-neox-20b": ModelType.PT,
+    "EleutherAI/pythia-1.4b-deduped": ModelType.PT,
+    "EleutherAI/pythia-2.7b": ModelType.PT,
+    "EleutherAI/pythia-6.9b-deduped": ModelType.PT,
+    "EleutherAI/pythia-70m": ModelType.PT,
+    "EleutherAI/gpt-j-6b": ModelType.PT,
+    "EleutherAI/pythia-12b-deduped": ModelType.PT,
+    "EleutherAI/gpt-neo-1.3B": ModelType.PT,
+    "EleutherAI/pythia-410m-deduped": ModelType.PT,
+    "EleutherAI/pythia-160m-deduped": ModelType.PT,
+    "EleutherAI/polyglot-ko-12.8b": ModelType.PT,
+    "EleutherAI/pythia-12b": ModelType.PT,
+    "roneneldan/TinyStories-33M": ModelType.PT,
+    "roneneldan/TinyStories-28M": ModelType.PT,
+    "roneneldan/TinyStories-1M": ModelType.PT,
+    "roneneldan/TinyStories-8M": ModelType.PT,
+    "roneneldan/TinyStories-3M": ModelType.PT,
+    "jerryjalapeno/nart-100k-7b": ModelType.FT,
+    "lmsys/vicuna-13b-v1.3": ModelType.IFT,
+    "lmsys/vicuna-7b-v1.3": ModelType.IFT,
+    "lmsys/vicuna-13b-v1.1": ModelType.IFT,
+    "lmsys/vicuna-13b-delta-v1.1": ModelType.IFT,
+    "lmsys/vicuna-7b-delta-v1.1": ModelType.IFT,
+    "abhiramtirumala/DialoGPT-sarcastic-medium": ModelType.FT,
+    "haonan-li/bactrian-x-llama-13b-merged": ModelType.IFT,
+    "Gryphe/MythoLogic-13b": ModelType.IFT,
+    "Gryphe/MythoBoros-13b": ModelType.IFT,
+    "pillowtalks-ai/delta13b": ModelType.FT,
+    "wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard": ModelType.FT,
+    "bigscience/bloom-7b1": ModelType.PT,
+    "bigcode/tiny_starcoder_py": ModelType.PT,
+    "bigcode/starcoderplus": ModelType.FT,
+    "bigcode/gpt_bigcode-santacoder": ModelType.PT,
+    "bigcode/starcoder": ModelType.PT,
+    "Open-Orca/OpenOrca-Preview1-13B": ModelType.IFT,
+    "microsoft/DialoGPT-large": ModelType.FT,
+    "microsoft/DialoGPT-small": ModelType.FT,
+    "microsoft/DialoGPT-medium": ModelType.FT,
+    "microsoft/CodeGPT-small-py": ModelType.FT,
+    "Tincando/fiction_story_generator": ModelType.FT,
+    "Pirr/pythia-13b-deduped-green_devil": ModelType.FT,
+    "Aeala/GPT4-x-AlpacaDente2-30b": ModelType.FT,
+    "Aeala/GPT4-x-AlpacaDente-30b": ModelType.FT,
+    "Aeala/GPT4-x-Alpasta-13b": ModelType.FT,
+    "Aeala/VicUnlocked-alpaca-30b": ModelType.IFT,
+    "Tap-M/Luna-AI-Llama2-Uncensored": ModelType.FT,
+    "illuin/test-custom-llama": ModelType.FT,
+    "dvruette/oasst-llama-13b-2-epochs": ModelType.FT,
+    "dvruette/oasst-gpt-neox-20b-1000-steps": ModelType.FT,
+    "dvruette/llama-13b-pretrained-dropout": ModelType.PT,
+    "dvruette/llama-13b-pretrained": ModelType.PT,
+    "dvruette/llama-13b-pretrained-sft-epoch-1": ModelType.FT,
+    "dvruette/llama-13b-pretrained-sft-do2": ModelType.FT,
+    "dvruette/oasst-gpt-neox-20b-3000-steps": ModelType.FT,
+    "dvruette/oasst-pythia-12b-pretrained-sft": ModelType.FT,
+    "dvruette/oasst-pythia-6.9b-4000-steps": ModelType.FT,
+    "dvruette/gpt-neox-20b-full-precision": ModelType.FT,
+    "dvruette/oasst-llama-13b-1000-steps": ModelType.FT,
+    "openlm-research/open_llama_7b_700bt_preview": ModelType.PT,
+    "openlm-research/open_llama_7b": ModelType.PT,
+    "openlm-research/open_llama_7b_v2": ModelType.PT,
+    "openlm-research/open_llama_3b": ModelType.PT,
+    "openlm-research/open_llama_13b": ModelType.PT,
+    "openlm-research/open_llama_3b_v2": ModelType.PT,
+    "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged": ModelType.IFT,
+    "GeorgiaTechResearchInstitute/galpaca-30b": ModelType.IFT,
+    "GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct": ModelType.IFT,
+    "databricks/dolly-v2-7b": ModelType.IFT,
+    "databricks/dolly-v2-3b": ModelType.IFT,
+    "databricks/dolly-v2-12b": ModelType.IFT,
+    "Rachneet/gpt2-xl-alpaca": ModelType.FT,
+    "Locutusque/gpt2-conversational-or-qa": ModelType.FT,
+    "psyche/kogpt": ModelType.FT,
+    "NbAiLab/nb-gpt-j-6B-alpaca": ModelType.IFT,
+    "Mikael110/llama-2-7b-guanaco-fp16": ModelType.FT,
+    "Mikael110/llama-2-13b-guanaco-fp16": ModelType.FT,
+    "Fredithefish/CrimsonPajama": ModelType.IFT,
+    "Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K": ModelType.FT,
+    "Fredithefish/ScarletPajama-3B-HF": ModelType.FT,
+    "Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4": ModelType.IFT,
+    "acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1": ModelType.IFT,
+    "eachadea/vicuna-13b-1.1": ModelType.FT,
+    "eachadea/vicuna-7b-1.1": ModelType.FT,
+    "eachadea/vicuna-13b": ModelType.FT,
+    "openaccess-ai-collective/wizard-mega-13b": ModelType.IFT,
+    "openaccess-ai-collective/manticore-13b": ModelType.IFT,
+    "openaccess-ai-collective/manticore-30b-chat-pyg-alpha": ModelType.IFT,
+    "openaccess-ai-collective/minotaur-13b": ModelType.IFT,
+    "openaccess-ai-collective/minotaur-13b-fixed": ModelType.IFT,
+    "openaccess-ai-collective/hippogriff-30b-chat": ModelType.IFT,
+    "openaccess-ai-collective/manticore-13b-chat-pyg": ModelType.IFT,
+    "pythainlp/wangchanglm-7.5B-sft-enth": ModelType.IFT,
+    "pythainlp/wangchanglm-7.5B-sft-en-sharded": ModelType.IFT,
+    "euclaise/gpt-neox-122m-minipile-digits": ModelType.FT,
+    "stabilityai/StableBeluga1-Delta": ModelType.IFT,
+    "stabilityai/stablelm-tuned-alpha-7b": ModelType.IFT,
+    "stabilityai/StableBeluga2": ModelType.IFT,
+    "stabilityai/StableBeluga-13B": ModelType.IFT,
+    "stabilityai/StableBeluga-7B": ModelType.IFT,
+    "stabilityai/stablelm-base-alpha-7b": ModelType.PT,
+    "stabilityai/stablelm-base-alpha-3b": ModelType.PT,
+    "stabilityai/stablelm-tuned-alpha-3b": ModelType.IFT,
+    "alibidaran/medical_transcription_generator": ModelType.FT,
+    "CalderaAI/30B-Lazarus": ModelType.IFT,
+    "CalderaAI/13B-BlueMethod": ModelType.IFT,
+    "CalderaAI/13B-Ouroboros": ModelType.IFT,
+    "KoboldAI/OPT-13B-Erebus": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Janeway": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Shinen": ModelType.FT,
+    "KoboldAI/fairseq-dense-2.7B": ModelType.PT,
+    "KoboldAI/OPT-6B-nerys-v2": ModelType.FT,
+    "KoboldAI/GPT-NeoX-20B-Skein": ModelType.FT,
+    "KoboldAI/PPO_Pygway-6b-Mix": ModelType.FT,
+    "KoboldAI/fairseq-dense-6.7B": ModelType.PT,
+    "KoboldAI/fairseq-dense-125M": ModelType.PT,
+    "KoboldAI/OPT-13B-Nerybus-Mix": ModelType.FT,
+    "KoboldAI/OPT-2.7B-Erebus": ModelType.FT,
+    "KoboldAI/OPT-350M-Nerys-v2": ModelType.FT,
+    "KoboldAI/OPT-2.7B-Nerys-v2": ModelType.FT,
+    "KoboldAI/OPT-2.7B-Nerybus-Mix": ModelType.FT,
+    "KoboldAI/OPT-13B-Nerys-v2": ModelType.FT,
+    "KoboldAI/GPT-NeoX-20B-Erebus": ModelType.FT,
+    "KoboldAI/OPT-6.7B-Erebus": ModelType.FT,
+    "KoboldAI/fairseq-dense-355M": ModelType.PT,
+    "KoboldAI/OPT-6.7B-Nerybus-Mix": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Adventure": ModelType.FT,
+    "KoboldAI/OPT-350M-Erebus": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Skein": ModelType.FT,
+    "KoboldAI/OPT-30B-Erebus": ModelType.FT,
+    "klosax/pythia-160m-deduped-step92k-193bt": ModelType.PT,
+    "klosax/open_llama_3b_350bt_preview": ModelType.PT,
+    "klosax/openllama-3b-350bt": ModelType.PT,
+    "klosax/pythia-70m-deduped-step44k-92bt": ModelType.PT,
+    "klosax/open_llama_13b_600bt_preview": ModelType.PT,
+    "klosax/open_llama_7b_400bt_preview": ModelType.PT,
+    "kfkas/Llama-2-ko-7b-Chat": ModelType.IFT,
+    "WeOpenML/Alpaca-7B-v1": ModelType.IFT,
+    "WeOpenML/PandaLM-Alpaca-7B-v1": ModelType.IFT,
+    "TFLai/gpt2-turkish-uncased": ModelType.FT,
+    "ehartford/WizardLM-13B-Uncensored": ModelType.IFT,
+    "ehartford/dolphin-llama-13b": ModelType.IFT,
+    "ehartford/Wizard-Vicuna-30B-Uncensored": ModelType.FT,
+    "ehartford/WizardLM-30B-Uncensored": ModelType.IFT,
+    "ehartford/Wizard-Vicuna-13B-Uncensored": ModelType.FT,
+    "ehartford/WizardLM-7B-Uncensored": ModelType.IFT,
+    "ehartford/based-30b": ModelType.FT,
+    "ehartford/Wizard-Vicuna-7B-Uncensored": ModelType.FT,
+    "wahaha1987/llama_7b_sharegpt94k_fastchat": ModelType.FT,
+    "wahaha1987/llama_13b_sharegpt94k_fastchat": ModelType.FT,
+    "OpenAssistant/oasst-sft-1-pythia-12b": ModelType.FT,
+    "OpenAssistant/stablelm-7b-sft-v7-epoch-3": ModelType.IFT,
+    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5": ModelType.FT,
+    "OpenAssistant/pythia-12b-sft-v8-2.5k-steps": ModelType.IFT,
+    "OpenAssistant/pythia-12b-sft-v8-7k-steps": ModelType.IFT,
+    "OpenAssistant/pythia-12b-pre-v8-12.5k-steps": ModelType.IFT,
+    "OpenAssistant/llama2-13b-orca-8k-3319": ModelType.IFT,
+    "junelee/wizard-vicuna-13b": ModelType.FT,
+    "BreadAi/gpt-YA-1-1_160M": ModelType.PT,
+    "BreadAi/MuseCan": ModelType.PT,
+    "BreadAi/MusePy-1-2": ModelType.PT,
+    "BreadAi/DiscordPy": ModelType.PT,
+    "BreadAi/PM_modelV2": ModelType.PT,
+    "BreadAi/gpt-Youtube": ModelType.PT,
+    "BreadAi/StoryPy": ModelType.FT,
+    "julianweng/Llama-2-7b-chat-orcah": ModelType.FT,
+    "AGI-inc/lora_moe_7b_baseline": ModelType.FT,
+    "AGI-inc/lora_moe_7b": ModelType.FT,
+    "togethercomputer/GPT-NeoXT-Chat-Base-20B": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Chat-7B-v0.1": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-7B-Base": ModelType.PT,
+    "togethercomputer/RedPajama-INCITE-7B-Instruct": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Base-3B-v1": ModelType.PT,
+    "togethercomputer/Pythia-Chat-Base-7B": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Base-7B-v0.1": ModelType.PT,
+    "togethercomputer/GPT-JT-6B-v1": ModelType.IFT,
+    "togethercomputer/GPT-JT-6B-v0": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Chat-3B-v1": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-7B-Chat": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Instruct-3B-v1": ModelType.IFT,
+    "Writer/camel-5b-hf": ModelType.IFT,
+    "Writer/palmyra-base": ModelType.PT,
+    "MBZUAI/LaMini-GPT-1.5B": ModelType.IFT,
+    "MBZUAI/lamini-cerebras-111m": ModelType.IFT,
+    "MBZUAI/lamini-neo-1.3b": ModelType.IFT,
+    "MBZUAI/lamini-cerebras-1.3b": ModelType.IFT,
+    "MBZUAI/lamini-cerebras-256m": ModelType.IFT,
+    "MBZUAI/LaMini-GPT-124M": ModelType.IFT,
+    "MBZUAI/lamini-neo-125m": ModelType.IFT,
+    "TehVenom/DiffMerge-DollyGPT-Pygmalion": ModelType.FT,
+    "TehVenom/PPO_Shygmalion-6b": ModelType.FT,
+    "TehVenom/Dolly_Shygmalion-6b-Dev_V8P2": ModelType.FT,
+    "TehVenom/Pygmalion_AlpacaLora-7b": ModelType.FT,
+    "TehVenom/PPO_Pygway-V8p4_Dev-6b": ModelType.FT,
+    "TehVenom/Dolly_Malion-6b": ModelType.FT,
+    "TehVenom/PPO_Shygmalion-V8p4_Dev-6b": ModelType.FT,
+    "TehVenom/ChanMalion": ModelType.FT,
+    "TehVenom/GPT-J-Pyg_PPO-6B": ModelType.IFT,
+    "TehVenom/Pygmalion-13b-Merged": ModelType.FT,
+    "TehVenom/Metharme-13b-Merged": ModelType.IFT,
+    "TehVenom/Dolly_Shygmalion-6b": ModelType.FT,
+    "TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4": ModelType.IFT,
+    "georgesung/llama2_7b_chat_uncensored": ModelType.FT,
+    "vicgalle/gpt2-alpaca": ModelType.IFT,
+    "vicgalle/alpaca-7b": ModelType.FT,
+    "vicgalle/gpt2-alpaca-gpt4": ModelType.IFT,
+    "facebook/opt-350m": ModelType.PT,
+    "facebook/opt-125m": ModelType.PT,
+    "facebook/xglm-4.5B": ModelType.PT,
+    "facebook/opt-2.7b": ModelType.PT,
+    "facebook/opt-6.7b": ModelType.PT,
+    "facebook/galactica-30b": ModelType.PT,
+    "facebook/opt-13b": ModelType.PT,
+    "facebook/opt-66b": ModelType.PT,
+    "facebook/xglm-7.5B": ModelType.PT,
+    "facebook/xglm-564M": ModelType.PT,
+    "facebook/opt-30b": ModelType.PT,
+    "golaxy/gogpt-7b": ModelType.FT,
+    "golaxy/gogpt2-7b": ModelType.FT,
+    "golaxy/gogpt-7b-bloom": ModelType.FT,
+    "golaxy/gogpt-3b-bloom": ModelType.FT,
+    "psmathur/orca_mini_v2_7b": ModelType.IFT,
+    "psmathur/orca_mini_7b": ModelType.IFT,
+    "psmathur/orca_mini_3b": ModelType.IFT,
+    "psmathur/orca_mini_v2_13b": ModelType.IFT,
+    "gpt2-xl": ModelType.PT,
+    "lxe/Cerebras-GPT-2.7B-Alpaca-SP": ModelType.FT,
+    "Monero/Manticore-13b-Chat-Pyg-Guanaco": ModelType.FT,
+    "Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b": ModelType.IFT,
+    "Monero/WizardLM-13b-OpenAssistant-Uncensored": ModelType.IFT,
+    "Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b": ModelType.IFT,
+    "jzjiao/opt-1.3b-rlhf": ModelType.FT,
+    "HuggingFaceH4/starchat-beta": ModelType.IFT,
+    "KnutJaegersberg/gpt-2-xl-EvolInstruct": ModelType.IFT,
+    "KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct": ModelType.IFT,
+    "KnutJaegersberg/galactica-orca-wizardlm-1.3b": ModelType.IFT,
+    "openchat/openchat_8192": ModelType.IFT,
+    "openchat/openchat_v2": ModelType.IFT,
+    "openchat/openchat_v2_w": ModelType.IFT,
+    "ausboss/llama-13b-supercot": ModelType.IFT,
+    "ausboss/llama-30b-supercot": ModelType.IFT,
+    "Neko-Institute-of-Science/metharme-7b": ModelType.IFT,
+    "Neko-Institute-of-Science/pygmalion-7b": ModelType.FT,
+    "SebastianSchramm/Cerebras-GPT-111M-instruction": ModelType.IFT,
+    "victor123/WizardLM-13B-1.0": ModelType.IFT,
+    "OpenBuddy/openbuddy-openllama-13b-v7-fp16": ModelType.FT,
+    "OpenBuddy/openbuddy-llama2-13b-v8.1-fp16": ModelType.FT,
+    "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16": ModelType.FT,
+    "baichuan-inc/Baichuan-7B": ModelType.PT,
+    "tiiuae/falcon-40b-instruct": ModelType.IFT,
+    "tiiuae/falcon-40b": ModelType.PT,
+    "tiiuae/falcon-7b": ModelType.PT,
+    "YeungNLP/firefly-llama-13b": ModelType.FT,
+    "YeungNLP/firefly-llama-13b-v1.2": ModelType.FT,
+    "YeungNLP/firefly-llama2-13b": ModelType.FT,
+    "YeungNLP/firefly-ziya-13b": ModelType.FT,
+    "shaohang/Sparse0.5_OPT-1.3": ModelType.FT,
+    "xzuyn/Alpacino-SuperCOT-13B": ModelType.IFT,
+    "xzuyn/MedicWizard-7B": ModelType.FT,
+    "xDAN-AI/xDAN_13b_l2_lora": ModelType.FT,
+    "beomi/KoAlpaca-Polyglot-5.8B": ModelType.FT,
+    "beomi/llama-2-ko-7b": ModelType.IFT,
+    "Salesforce/codegen-6B-multi": ModelType.PT,
+    "Salesforce/codegen-16B-nl": ModelType.PT,
+    "Salesforce/codegen-6B-nl": ModelType.PT,
+    "ai-forever/rugpt3large_based_on_gpt2": ModelType.FT,
+    "gpt2-large": ModelType.PT,
+    "frank098/orca_mini_3b_juniper": ModelType.FT,
+    "frank098/WizardLM_13B_juniper": ModelType.FT,
+    "FPHam/Free_Sydney_13b_HF": ModelType.FT,
+    "huggingface/llama-13b": ModelType.PT,
+    "huggingface/llama-7b": ModelType.PT,
+    "huggingface/llama-65b": ModelType.PT,
+    "huggingface/llama-30b": ModelType.PT,
+    "Henk717/chronoboros-33B": ModelType.IFT,
+    "jondurbin/airoboros-13b-gpt4-1.4": ModelType.IFT,
+    "jondurbin/airoboros-7b": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.1": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.2": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.3": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.4": ModelType.IFT,
+    "jondurbin/airoboros-l2-7b-gpt4-1.4.1": ModelType.IFT,
+    "jondurbin/airoboros-l2-13b-gpt4-1.4.1": ModelType.IFT,
+    "jondurbin/airoboros-l2-70b-gpt4-1.4.1": ModelType.IFT,
+    "jondurbin/airoboros-13b": ModelType.IFT,
+    "jondurbin/airoboros-33b-gpt4-1.4": ModelType.IFT,
+    "jondurbin/airoboros-33b-gpt4-1.2": ModelType.IFT,
+    "jondurbin/airoboros-65b-gpt4-1.2": ModelType.IFT,
+    "ariellee/SuperPlatty-30B": ModelType.IFT,
+    "danielhanchen/open_llama_3b_600bt_preview": ModelType.FT,
+    "cerebras/Cerebras-GPT-256M": ModelType.PT,
+    "cerebras/Cerebras-GPT-1.3B": ModelType.PT,
+    "cerebras/Cerebras-GPT-13B": ModelType.PT,
+    "cerebras/Cerebras-GPT-2.7B": ModelType.PT,
+    "cerebras/Cerebras-GPT-111M": ModelType.PT,
+    "cerebras/Cerebras-GPT-6.7B": ModelType.PT,
+    "Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf": ModelType.RL,
+    "Yhyu13/llama-30B-hf-openassitant": ModelType.FT,
+    "NousResearch/Nous-Hermes-Llama2-13b": ModelType.IFT,
+    "NousResearch/Nous-Hermes-llama-2-7b": ModelType.IFT,
+    "NousResearch/Redmond-Puffin-13B": ModelType.IFT,
+    "NousResearch/Nous-Hermes-13b": ModelType.IFT,
+    "project-baize/baize-v2-7b": ModelType.IFT,
+    "project-baize/baize-v2-13b": ModelType.IFT,
+    "LLMs/WizardLM-13B-V1.0": ModelType.FT,
+    "LLMs/AlpacaGPT4-7B-elina": ModelType.FT,
+    "wenge-research/yayi-7b": ModelType.FT,
+    "wenge-research/yayi-7b-llama2": ModelType.FT,
+    "wenge-research/yayi-13b-llama2": ModelType.FT,
+    "yhyhy3/open_llama_7b_v2_med_instruct": ModelType.IFT,
+    "llama-anon/instruct-13b": ModelType.IFT,
+    "huggingtweets/jerma985": ModelType.FT,
+    "huggingtweets/gladosystem": ModelType.FT,
+    "huggingtweets/bladeecity-jerma985": ModelType.FT,
+    "huggyllama/llama-13b": ModelType.PT,
+    "huggyllama/llama-65b": ModelType.PT,
+    "FabbriSimo01/Facebook_opt_1.3b_Quantized": ModelType.PT,
+    "upstage/Llama-2-70b-instruct": ModelType.IFT,
+    "upstage/Llama-2-70b-instruct-1024": ModelType.IFT,
+    "upstage/llama-65b-instruct": ModelType.IFT,
+    "upstage/llama-30b-instruct-2048": ModelType.IFT,
+    "upstage/llama-30b-instruct": ModelType.IFT,
+    "WizardLM/WizardLM-13B-1.0": ModelType.IFT,
+    "WizardLM/WizardLM-13B-V1.1": ModelType.IFT,
+    "WizardLM/WizardLM-13B-V1.2": ModelType.IFT,
+    "WizardLM/WizardLM-30B-V1.0": ModelType.IFT,
+    "WizardLM/WizardCoder-15B-V1.0": ModelType.IFT,
+    "gpt2": ModelType.PT,
+    "keyfan/vicuna-chinese-replication-v1.1": ModelType.IFT,
+    "nthngdy/pythia-owt2-70m-100k": ModelType.FT,
+    "nthngdy/pythia-owt2-70m-50k": ModelType.FT,
+    "quantumaikr/KoreanLM-hf": ModelType.FT,
+    "quantumaikr/open_llama_7b_hf": ModelType.FT,
+    "quantumaikr/QuantumLM-70B-hf": ModelType.IFT,
+    "MayaPH/FinOPT-Lincoln": ModelType.FT,
+    "MayaPH/FinOPT-Franklin": ModelType.FT,
+    "MayaPH/GodziLLa-30B": ModelType.IFT,
+    "MayaPH/GodziLLa-30B-plus": ModelType.IFT,
+    "MayaPH/FinOPT-Washington": ModelType.FT,
+    "ogimgio/gpt-neo-125m-neurallinguisticpioneers": ModelType.FT,
+    "layoric/llama-2-13b-code-alpaca": ModelType.FT,
+    "CobraMamba/mamba-gpt-3b": ModelType.FT,
+    "CobraMamba/mamba-gpt-3b-v2": ModelType.FT,
+    "CobraMamba/mamba-gpt-3b-v3": ModelType.FT,
+    "timdettmers/guanaco-33b-merged": ModelType.FT,
+    "elinas/chronos-33b": ModelType.IFT,
+    "heegyu/RedTulu-Uncensored-3B-0719": ModelType.IFT,
+    "heegyu/WizardVicuna-Uncensored-3B-0719": ModelType.IFT,
+    "heegyu/WizardVicuna-3B-0719": ModelType.IFT,
+    "meta-llama/Llama-2-7b-chat-hf": ModelType.RL,
+    "meta-llama/Llama-2-7b-hf": ModelType.PT,
+    "meta-llama/Llama-2-13b-chat-hf": ModelType.RL,
+    "meta-llama/Llama-2-13b-hf": ModelType.PT,
+    "meta-llama/Llama-2-70b-chat-hf": ModelType.RL,
+    "meta-llama/Llama-2-70b-hf": ModelType.PT,
+    "xhyi/PT_GPTNEO350_ATG": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-20b": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt": ModelType.FT,
+    "h2oai/h2ogpt-oig-oasst1-512-6_9b": ModelType.IFT,
+    "h2oai/h2ogpt-oasst1-512-12b": ModelType.IFT,
+    "h2oai/h2ogpt-oig-oasst1-256-6_9b": ModelType.IFT,
+    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt": ModelType.FT,
+    "h2oai/h2ogpt-oasst1-512-20b": ModelType.IFT,
+    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-12b": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-multilang-1024-20b": ModelType.FT,
+    "bofenghuang/vigogne-13b-instruct": ModelType.IFT,
+    "bofenghuang/vigogne-13b-chat": ModelType.FT,
+    "bofenghuang/vigogne-2-7b-instruct": ModelType.IFT,
+    "bofenghuang/vigogne-7b-instruct": ModelType.IFT,
+    "bofenghuang/vigogne-7b-chat": ModelType.FT,
+    "Vmware/open-llama-7b-v2-open-instruct": ModelType.IFT,
+    "VMware/open-llama-0.7T-7B-open-instruct-v1.1": ModelType.IFT,
+    "ewof/koishi-instruct-3b": ModelType.IFT,
+    "gywy/llama2-13b-chinese-v1": ModelType.FT,
+    "GOAT-AI/GOAT-7B-Community": ModelType.FT,
+    "psyche/kollama2-7b": ModelType.FT,
+    "TheTravellingEngineer/llama2-7b-hf-guanaco": ModelType.FT,
+    "beaugogh/pythia-1.4b-deduped-sharegpt": ModelType.FT,
+    "augtoma/qCammel-70-x": ModelType.IFT,
+    "Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload": ModelType.IFT,
+    "anhnv125/pygmalion-6b-roleplay": ModelType.FT,
+    "64bits/LexPodLM-13B": ModelType.FT,
+}
+
+
+def model_type_from_str(type):
+    if "fine-tuned" in type or "🔶" in type:
+        return ModelType.FT
+    if "pretrained" in type or "🟢" in type:
+        return ModelType.PT
+    if "RL-tuned" in type or "🟦" in type:
+        return ModelType.RL
+    if "instruction-tuned" in type or "⭕" in type:
+        return ModelType.IFT
+    return ModelType.Unknown

src/display_models/read_results.py

@@ -0,0 +1,153 @@
+import json
+import os
+from dataclasses import dataclass
+from typing import Dict, List, Tuple
+
+import dateutil
+import numpy as np
+
+from src.display_models.utils import AutoEvalColumn, make_clickable_model
+
+METRICS = ["acc_norm", "acc_norm", "acc", "mc2"]
+BENCHMARKS = ["arc:challenge", "hellaswag", "hendrycksTest", "truthfulqa:mc", "commongen", "ethicalverification"]
+BENCH_TO_NAME = {
+    "arc:challenge": AutoEvalColumn.arc.name,
+    "hellaswag": AutoEvalColumn.hellaswag.name,
+    "hendrycksTest": AutoEvalColumn.mmlu.name,
+    "truthfulqa:mc": AutoEvalColumn.truthfulqa.name,
+    "commongen": AutoEvalColumn.commongen.name,
+    "ethicalverification": AutoEvalColumn.ethicalverification.name,
+}
+
+
+@dataclass
+class EvalResult:
+    eval_name: str
+    org: str
+    model: str
+    revision: str
+    results: dict
+    precision: str = ""
+    model_type: str = ""
+    weight_type: str = ""
+
+    def to_dict(self):
+        from src.load_from_hub import is_model_on_hub
+
+        if self.org is not None:
+            base_model = f"{self.org}/{self.model}"
+        else:
+            base_model = f"{self.model}"
+        data_dict = {}
+
+        data_dict["eval_name"] = self.eval_name  # not a column, just a save name
+        data_dict["weight_type"] = self.weight_type  # not a column, just a save name
+        data_dict[AutoEvalColumn.precision.name] = self.precision
+        data_dict[AutoEvalColumn.model_type.name] = self.model_type
+        data_dict[AutoEvalColumn.model.name] = make_clickable_model(base_model)
+        data_dict[AutoEvalColumn.dummy.name] = base_model
+        data_dict[AutoEvalColumn.revision.name] = self.revision
+        data_dict[AutoEvalColumn.average.name] = sum([v for k, v in self.results.items()]) / 4.0
+        data_dict[AutoEvalColumn.still_on_hub.name] = (
+            is_model_on_hub(base_model, self.revision)[0] or base_model == "baseline"
+        )
+
+        for benchmark in BENCHMARKS:
+            if benchmark not in self.results.keys():
+                self.results[benchmark] = None
+
+        for k, v in BENCH_TO_NAME.items():
+            data_dict[v] = self.results[k]
+
+        return data_dict
+
+
+def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
+    with open(json_filepath) as fp:
+        data = json.load(fp)
+
+    for mmlu_k in ["harness|hendrycksTest-abstract_algebra|5", "hendrycksTest-abstract_algebra"]:
+        if mmlu_k in data["versions"] and data["versions"][mmlu_k] == 0:
+            return None, []  # we skip models with the wrong version
+
+    try:
+        config = data["config"]
+    except KeyError:
+        config = data["config_general"]
+    model = config.get("model_name", None)
+    if model is None:
+        model = config.get("model_args", None)
+
+    model_sha = config.get("model_sha", "")
+    model_split = model.split("/", 1)
+
+    precision = config.get("model_dtype")
+
+    model = model_split[-1]
+
+    if len(model_split) == 1:
+        org = None
+        model = model_split[0]
+        result_key = f"{model}_{model_sha}_{precision}"
+    else:
+        org = model_split[0]
+        model = model_split[1]
+        result_key = f"{org}_{model}_{model_sha}_{precision}"
+
+    eval_results = []
+    for benchmark, metric in zip(BENCHMARKS, METRICS):
+        accs = np.array([v.get(metric, None) for k, v in data["results"].items() if benchmark in k])
+        if accs.size == 0 or any([acc is None for acc in accs]):
+            continue
+        mean_acc = np.mean(accs) * 100.0
+        eval_results.append(
+            EvalResult(
+                eval_name=result_key,
+                org=org,
+                model=model,
+                revision=model_sha,
+                results={benchmark: mean_acc},
+                precision=precision,  # todo model_type=, weight_type=
+            )
+        )
+
+    return result_key, eval_results
+
+
+def get_eval_results() -> List[EvalResult]:
+    json_filepaths = []
+
+    for root, dir, files in os.walk("eval-results"):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        # store results by precision maybe?
+        try:
+            files.sort(key=lambda x: dateutil.parser.parse(x.split("_", 1)[-1][:-5]))
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        # up_to_date = files[-1]
+        for file in files:
+            json_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for json_filepath in json_filepaths:
+        result_key, results = parse_eval_result(json_filepath)
+        for eval_result in results:
+            if result_key in eval_results.keys():
+                eval_results[result_key].results.update(eval_result.results)
+            else:
+                eval_results[result_key] = eval_result
+
+    eval_results = [v for v in eval_results.values()]
+
+    return eval_results
+
+
+def get_eval_results_dicts() -> List[Dict]:
+    eval_results = get_eval_results()
+
+    return [e.to_dict() for e in eval_results]

src/display_models/utils.py

@@ -0,0 +1,148 @@
+import os
+from dataclasses import dataclass
+
+from huggingface_hub import HfApi
+
+API = HfApi()
+
+
+# These classes are for user facing column names, to avoid having to change them
+# all around the code when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+@dataclass(frozen=True)
+class AutoEvalColumn:  # Auto evals column
+    model_type_symbol = ColumnContent("T", "str", True)
+    model = ColumnContent("Model", "markdown", True)
+    average = ColumnContent("Average ⬆️", "number", True)
+    arc = ColumnContent("Ko-ARC", "number", True)
+    hellaswag = ColumnContent("Ko-HellaSwag", "number", True)
+    mmlu = ColumnContent("Ko-MMLU", "number", True)
+    truthfulqa = ColumnContent("Ko-TruthfulQA", "number", True)
+    commongen = ColumnContent("Ko-CommonGen", "number", True)
+    ethicalverification = ColumnContent("EthicalVerification", "number", True)
+    model_type = ColumnContent("Type", "str", False)
+    precision = ColumnContent("Precision", "str", False)  # , True)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    still_on_hub = ColumnContent("Available on the hub", "bool", False)
+    revision = ColumnContent("Model sha", "str", False, False)
+    dummy = ColumnContent(
+        "model_name_for_query", "str", True
+    )  # dummy col to implement search bar (hidden by custom CSS)
+
+
+@dataclass(frozen=True)
+class EloEvalColumn:  # Elo evals column
+    model = ColumnContent("Model", "markdown", True)
+    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
+    human_all = ColumnContent("Human (all)", "number", True)
+    human_instruct = ColumnContent("Human (instruct)", "number", True)
+    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+
+LLAMAS = [
+    "huggingface/llama-7b",
+    "huggingface/llama-13b",
+    "huggingface/llama-30b",
+    "huggingface/llama-65b",
+]
+
+
+KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
+VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
+OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"
+DOLLY_LINK = "https://huggingface.co/databricks/dolly-v2-12b"
+MODEL_PAGE = "https://huggingface.co/models"
+LLAMA_LINK = "https://ai.facebook.com/blog/large-language-model-llama-meta-ai/"
+VICUNA_LINK = "https://huggingface.co/CarperAI/stable-vicuna-13b-delta"
+ALPACA_LINK = "https://crfm.stanford.edu/2023/03/13/alpaca.html"
+
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+
+    if model_name in LLAMAS:
+        link = LLAMA_LINK
+        model_name = model_name.split("/")[1]
+    elif model_name == "HuggingFaceH4/stable-vicuna-13b-2904":
+        link = VICUNA_LINK
+        model_name = "stable-vicuna-13b"
+    elif model_name == "HuggingFaceH4/llama-7b-ift-alpaca":
+        link = ALPACA_LINK
+        model_name = "alpaca-13b"
+    if model_name == "dolly-12b":
+        link = DOLLY_LINK
+    elif model_name == "vicuna-13b":
+        link = VICUNA_LINK
+    elif model_name == "koala-13b":
+        link = KOALA_LINK
+    elif model_name == "oasst-12b":
+        link = OASST_LINK
+
+    details_model_name = model_name.replace("/", "__")
+    details_link = f"https://huggingface.co/datasets/open-llm-leaderboard/details_{details_model_name}"
+
+    if not bool(os.getenv("DEBUG", "False")):
+        # We only add these checks when not debugging, as they are extremely slow
+        print(f"details_link: {details_link}")
+        try:
+            check_path = list(
+                API.list_files_info(
+                    repo_id=f"open-llm-leaderboard/details_{details_model_name}",
+                    paths="README.md",
+                    repo_type="dataset",
+                )
+            )
+            print(f"check_path: {check_path}")
+        except Exception as err:
+            # No details repo for this model
+            print(f"No details repo for this model: {err}")
+            return model_hyperlink(link, model_name)
+
+    return model_hyperlink(link, model_name) + "  " + model_hyperlink(details_link, "📑")
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/load_from_hub.py

@@ -0,0 +1,151 @@
+import json
+import os
+
+import pandas as pd
+from huggingface_hub import Repository
+from transformers import AutoConfig
+from collections import defaultdict
+
+from src.assets.hardcoded_evals import baseline, gpt4_values, gpt35_values
+from src.display_models.get_model_metadata import apply_metadata
+from src.display_models.read_results import get_eval_results_dicts, make_clickable_model
+from src.display_models.utils import AutoEvalColumn, EvalQueueColumn, has_no_nan_values
+
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+
+
+def get_all_requested_models(requested_models_dir: str) -> set[str]:
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"): continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates
+
+
+def load_all_info_from_hub(QUEUE_REPO: str, RESULTS_REPO: str, QUEUE_PATH: str, RESULTS_PATH: str) -> list[Repository]:
+    eval_queue_repo = None
+    eval_results_repo = None
+    requested_models = None
+
+    print("Pulling evaluation requests and results.")
+
+    eval_queue_repo = Repository(
+        local_dir=QUEUE_PATH,
+        clone_from=QUEUE_REPO,
+        repo_type="dataset",
+    )
+    eval_queue_repo.git_pull()
+
+    eval_results_repo = Repository(
+        local_dir=RESULTS_PATH,
+        clone_from=RESULTS_REPO,
+        repo_type="dataset",
+    )
+    eval_results_repo.git_pull()
+
+    requested_models, users_to_submission_dates = get_all_requested_models("eval-queue")
+
+    return eval_queue_repo, requested_models, eval_results_repo, users_to_submission_dates
+
+
+def get_leaderboard_df(
+    eval_results: Repository, eval_results_private: Repository, cols: list, benchmark_cols: list
+) -> pd.DataFrame:
+    if eval_results:
+        print("Pulling evaluation results for the leaderboard.")
+        eval_results.git_pull()
+    if eval_results_private:
+        print("Pulling evaluation results for the leaderboard.")
+        eval_results_private.git_pull()
+
+    all_data = get_eval_results_dicts()
+
+    if not IS_PUBLIC:
+        all_data.append(gpt4_values)
+        all_data.append(gpt35_values)
+
+    all_data.append(baseline)
+    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
+
+    df = pd.DataFrame.from_records(all_data)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return df
+
+
+def get_evaluation_queue_df(
+    eval_queue: Repository, eval_queue_private: Repository, save_path: str, cols: list
+) -> list[pd.DataFrame]:
+    if eval_queue:
+        print("Pulling changes for the evaluation queue.")
+        eval_queue.git_pull()
+    if eval_queue_private:
+        print("Pulling changes for the evaluation queue.")
+        eval_queue_private.git_pull()
+
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]
+
+
+def is_model_on_hub(model_name: str, revision: str) -> bool:
+    try:
+        AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False)
+        return True, None
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+        )
+
+    except Exception as e:
+        print(f"Could not get the model config from the hub.: {e}")
+        return False, "was not found on hub!"

src/rate_limiting.py

@@ -0,0 +1,16 @@
+
+from datetime import datetime, timezone, timedelta
+
+
+def user_submission_permission(submission_name, users_to_submission_dates, rate_limit_period):
+    org_or_user, _ = submission_name.split("/")
+    if org_or_user not in users_to_submission_dates:
+        return 0
+    submission_dates = sorted(users_to_submission_dates[org_or_user])
+
+    time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
+
+    return len(submissions_after_timelimit)
+
+