add support for model submission (#9)

- add support for model submission (58cc5c3c2bd8921d517aa58e67e908643d882c5d)
- Update src/utils.py (05a0e4b85b396e5177116b204be1d2009f8c34cc)
- Update src/text_content.py (8672ed1cb0633c4841d31cc5e8515f37234080d6)

app.py

@@ -1,15 +1,30 @@
 # some code blocks are taken from https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/tree/main
+import json
+import os
+from datetime import datetime, timezone
+
 import gradio as gr
 import pandas as pd
+from huggingface_hub import HfApi
 
 from src.css_html import custom_css
-from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT
-from src.utils import (AutoEvalColumn, fields, make_clickable_names,
-                       plot_throughput)
-
+from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT, SUBMISSION_TEXT_2
+from src.utils import (
+    AutoEvalColumn,
+    fields,
+    is_model_on_hub,
+    make_clickable_names,
+    plot_throughput,
+    styled_error,
+    styled_message,
+)
 
+TOKEN = os.environ.get("HF_TOKEN", None)
+api = HfApi(TOKEN)
 df = pd.read_csv("data/code_eval_board.csv")
 
+QUEUE_REPO = "bigcode/evaluation-requests"
+EVAL_REQUESTS_PATH = "eval-queue"
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_LITE = [
@@ -20,6 +35,65 @@ TYPES_LITE = [
 ]
 
 
+def add_new_eval(
+    model: str,
+    revision: str,
+    precision: str,
+    model_type: str,
+):
+    precision = precision
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # check the model actually exists before adding the eval
+    if revision == "":
+        revision = "main"
+
+    model_on_hub, error = is_model_on_hub(model, revision)
+    if not model_on_hub:
+        return styled_error(f'Model "{model}" {error}')
+
+    print("adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "revision": revision,
+        "precision": precision,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type.split(" ")[1],
+    }
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{precision}.json"
+    print(f"Saving eval request to {out_path}")
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    api.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # remove the local file
+    os.remove(out_path)
+
+    return styled_message("Your request has been submitted to the evaluation queue!\n")
+
+
 def select_columns(df, columns):
     always_here_cols = [
         AutoEvalColumn.model_type_symbol.name,
@@ -56,8 +130,9 @@ with demo:
             """<div style="text-align: center;"><h1> ⭐ Big <span style='color: #e6b800;'>Code</span> Models <span style='color: #e6b800;'>Leaderboard</span></h1></div>\
             <br>\
             <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/optimum/llm-perf-leaderboard">🤗 Open LLM-Perf Leaderboard 🏋️</a>, we compare performance of base multilingual code generation models on <a href="https://huggingface.co/datasets/openai_humaneval">HumanEval</a> benchmark and <a href="https://huggingface.co/datasets/nuprl/MultiPL-E">MultiPL-E</a>. We also measure throughput and provide\
-            information about the models. We only compare open pre-trained multilingual code models, that people can start from as base models for their trainings.</p>"""
-        , elem_classes="markdown-text") 
+            information about the models. We only compare open pre-trained multilingual code models, that people can start from as base models for their trainings.</p>""",
+            elem_classes="markdown-text",
+        )
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.Column():
@@ -142,13 +217,16 @@ with demo:
                         [hidden_leaderboard_df, shown_columns],
                         leaderboard_df,
                     )
-                    gr.Markdown("""
+                    gr.Markdown(
+                        """
                     **Notes:**
                     - Win Rate represents how often a model outperforms other models in each language, averaged across all languages.
                     - The scores of instruction-tuned models might be significantly higher on humaneval-python than other languages because we use the instruction prompt format of this benchmark.
                     - For more details check the 📝 About section.
-                    """, elem_classes="markdown-text")      
-                            
+                    """,
+                        elem_classes="markdown-text",
+                    )
+
                 with gr.TabItem("📊 Performance Plot", id=1):
                     with gr.Row():
                         bs_1_plot = gr.components.Plot(
@@ -161,11 +239,51 @@ with demo:
                             elem_id="bs50-plot",
                             show_label=False,
                         )
-                    gr.Markdown("**Note:** Zero throughput on the right plot refers to OOM, for more details check the 📝 About section.", elem_classes="markdown-text") 
+                    gr.Markdown(
+                        "**Note:** Zero throughput on the right plot refers to OOM, for more details check the 📝 About section.",
+                        elem_classes="markdown-text",
+                    )
                 with gr.TabItem("📝 About", id=2):
                     gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
                 with gr.TabItem("Submit results 🚀", id=3):
                     gr.Markdown(SUBMISSION_TEXT)
+                    gr.Markdown(
+                        "## 📤  Submit your model here:", elem_classes="markdown-text"
+                    )
+                    with gr.Column():
+                        with gr.Row():
+                            model_name = gr.Textbox(label="Model name")
+                            revision_name = gr.Textbox(
+                                label="revision", placeholder="main"
+                            )
+                        with gr.Row():
+                            precision = gr.Dropdown(
+                                choices=[
+                                    "float16",
+                                    "bfloat16",
+                                    "8bit",
+                                    "4bit",
+                                ],
+                                label="Precision",
+                                multiselect=False,
+                                value="float16",
+                                interactive=True,
+                            )
+                            model_type = gr.Dropdown(
+                                choices=["🟢 base", "🔶 instruction-tuned"],
+                                label="Model type",
+                                multiselect=False,
+                                value=None,
+                                interactive=True,
+                            )
+                        submit_button = gr.Button("Submit Eval")
+                        submission_result = gr.Markdown()
+                        submit_button.click(
+                            add_new_eval,
+                            inputs=[model_name, revision_name, precision, model_type],
+                            outputs=[submission_result],
+                        )
+                        gr.Markdown(SUBMISSION_TEXT_2)
 
 
-demo.launch()
+demo.launch()

src/text_content.py

@@ -29,9 +29,35 @@ The growing number of code models released by the community necessitates a compr
 
 SUBMISSION_TEXT = """
 <h1 align="center">
-How to submit new results to the leaderboard?
+How to submit models/results to the leaderboard?
 </h1>
-We welcome the community to submit evaluation results of new models. These results will be added as non-verified, the authors are however required to upload their generations in case other members want to check.
+We welcome the community to submit evaluation results of new models. We also provide an experiental feature for submitting models that our team will evaluate on the 🤗 cluster.
+
+## Submitting Models (experimental feature)
+Inspired from the Open LLM Leaderboard, we welcome code models submission from the community that will be automatically evaluated. Please note that this is still an experimental feature.
+Below are some guidlines to follow before submitting your model:
+
+#### 1) Make sure you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+Note: make sure your model is public!
+Note: if your model needs `use_remote_code=True`, we do not support this option yet.
+#### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+#### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+#### 4) Fill up your model card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card.
+"""
+
+SUBMISSION_TEXT_2 = """
+## Sumbitting Results
+You also have the option for running evaluation yourself and submitting results. These results will be added as non-verified, the authors are however required to upload their generations in case other members want to check.
 
 ### 1 - Running Evaluation
 

src/utils.py

@@ -1,7 +1,7 @@
 # source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/utils_display.py
 from dataclasses import dataclass
 import plotly.graph_objects as go
-
+from transformers import AutoConfig
 
 # These classes are for user facing column names, to avoid having to change them
 # all around the code when a modif is needed
@@ -111,3 +111,39 @@ def plot_throughput(df, bs=1):
         yaxis_title="Average Code Score",
     )
     return fig
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)
+
+
+def is_model_on_hub(model_name: str, revision: str) -> bool:
+    try:
+        AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False)
+        return True, None
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+        )
+
+    except Exception as e:
+        print(f"Could not get the model config from the hub.: {e}")
+        return False, "was not found on hub!"