Add initial space version

app.py

@@ -0,0 +1,101 @@
+import os
+
+import gradio as gr
+
+from src.content import (INTRODUCTION_TEXT, INTRODUCTION_TITLE,
+                         LEADERBOARD_TEXT, LEADERBOARD_TITLE,
+                         SUBMISSION_TEXT_FILES, SUBMISSION_TEXT_INTRO,
+                         SUBMISSION_TEXT_METADATA, SUBMISSION_TEXT_SUBMIT,
+                         SUBMISSION_TEXT_TASK, SUBMISSION_TITLE)
+from src.get_results_for_task import get_results_for_task_stub
+from src.submission_uploader import SubmissionUploader
+from src.tasks import TASKS_DESCRIPTIONS, TASKS_PRETTY, TASKS_PRETTY_REVERSE
+
+submission_uploader = SubmissionUploader(os.environ["DATASET_ID"])
+
+
+with gr.Blocks() as demo:
+    gr.HTML(INTRODUCTION_TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    gr.HTML(LEADERBOARD_TITLE)
+    gr.Markdown(LEADERBOARD_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs():
+        for task in TASKS_PRETTY_REVERSE:
+            with gr.TabItem(task):
+                with gr.Row():
+                    gr.Markdown(TASKS_DESCRIPTIONS[task])
+
+                leaderboard_table = gr.components.Dataframe(
+                    value=get_results_for_task_stub(task), interactive=False
+                )
+
+    gr.HTML(SUBMISSION_TITLE)
+    gr.Markdown(SUBMISSION_TEXT_INTRO, elem_classes="markdown-text")
+
+    with gr.Accordion("🚀 Submit new results"):
+        gr.Markdown(SUBMISSION_TEXT_TASK, elem_classes="markdown-text")
+        task = gr.Radio(TASKS_PRETTY_REVERSE.keys(), label="Task")
+
+        gr.Markdown(SUBMISSION_TEXT_METADATA, elem_classes="markdown-text")
+        with gr.Row():
+            with gr.Column():
+                model_folder_textbox = gr.Textbox(
+                    label="Model Folder",
+                    placeholder="How to call a folder related to this submission in our results dataset.",
+                )
+                model_name_textbox = gr.Textbox(
+                    label="Model Name",
+                    placeholder="How to display model's name on the leaderboard.",
+                )
+                model_availability_textbox = gr.Textbox(
+                    label="Availability",
+                    placeholder="Information about the model's availability and licensing.",
+                )
+                context_size_textbox = gr.Textbox(
+                    label="Context Size",
+                    placeholder="Context size (in tokens) used for the submission.",
+                )
+            with gr.Column():
+                submitted_by_textbox = gr.Textbox(
+                    label="Submitted By",
+                    placeholder="Who submitted the model, how it will be displayed on the leaderboard.",
+                )
+                contact_textbox = gr.Textbox(
+                    label="Contact Information",
+                    placeholder="How Long Code Arena team can contact you in case of any questions (won't go to public dataset).",
+                )
+                comment_textbox = gr.Textbox(
+                    label="Comment",
+                    placeholder="Any comments you have for Long Code Arena team (optional, won't go to public dataset).",
+                )
+                url_textbox = gr.Textbox(
+                    label="Relevant URLs",
+                    placeholder="URLs to relevant resources (preprint/blogpost/code/etc.) with "
+                    "additional details about your submission.",
+                )
+
+        gr.Markdown(SUBMISSION_TEXT_FILES, elem_classes="markdown-text")
+        file_output = gr.File(file_count="multiple")
+
+        gr.Markdown(SUBMISSION_TEXT_SUBMIT, elem_classes="markdown-text")
+        submit_button = gr.Button("Submit")
+        submission_result = gr.Markdown()
+        submit_button.click(
+            submission_uploader.upload_files,
+            [
+                task,
+                model_folder_textbox,
+                model_name_textbox,
+                model_availability_textbox,
+                url_textbox,
+                context_size_textbox,
+                submitted_by_textbox,
+                file_output,
+            ],
+            submission_result,
+        )
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1 @@
+huggingface_hub

src/content.py

@@ -0,0 +1,31 @@
+# ================================
+# =            ABOUT             =
+# ================================
+INTRODUCTION_TITLE = """<h1 align="center">🏟️ Long Code Arena</h1>"""
+
+INTRODUCTION_TEXT = """🏟️ Long Code Arena is a benchmark of code-related tasks with large contexts, up to a whole code repository.
+It currently spans six different tasks."""
+
+# ================================
+# =         LEADERBOARD          =
+# ================================
+LEADERBOARD_TITLE = '<h2 align="center">🏅Leaderboard</h2>'
+
+LEADERBOARD_TEXT = """Raw results from the leaderboard are available in 🤗 [JetBrains-Research/lca-results](https://huggingface.co/datasets/JetBrains-Research/lca-results)."""
+
+# ================================
+# =          SUBMISSION          =
+# ================================
+SUBMISSION_TITLE = '<h2 align="center">📩 Make A Submission</h2>'
+
+SUBMISSION_TEXT_INTRO = """Use the form below to submit new results to 🏟️ Long Code Arena. If any problems arise, don't hesitate to contact us by email `TODO` or open a discussion 💛"""
+
+SUBMISSION_TEXT_TASK = """1. Select a task you want to submit results for."""
+
+SUBMISSION_TEXT_METADATA = """2. Fill in some metadata about your submission."""
+
+SUBMISSION_TEXT_FILES = """3. Attach one or more files with your model's predictions.
+    * If several files are attached, they will be treated as separate runs of the submitted model (e.g., with different seeds), and the metrics will be averaged across runs. For baselines provided by 🏟️ Long Code Arena Team, the results are averaged across 3 runs.
+    * Please, attach files in [JSONLines format](https://jsonlines.org/). For an example, check the predictions provided by 🏟️ Long Code Arena Team in  🤗 [JetBrains-Research/lca-results](https://huggingface.co/datasets/JetBrains-Research/lca-results). Make sure to include `"prediction"` and `"reference"` fields for each example, the rest are optional. 
+"""
+SUBMISSION_TEXT_SUBMIT = """All set! A new PR to 🤗 [JetBrains-Research/lca-results](https://huggingface.co/datasets/JetBrains-Research/lca-results) should be opened when you press "Submit" button. 🏟️ Long Code Arena Team will review it shortly, and the results will appear in the leaderboard."""

src/get_results_for_task.py

@@ -0,0 +1,33 @@
+import pandas as pd
+
+RESULTS_DATASET = "JetBrains-Research/lca-results"
+
+
+def get_results_for_task_stub(task: str) -> pd.DataFrame:
+    stub_df = pd.DataFrame(
+        [
+            {
+                "Model Name": "GPT-4",
+                "Availability": "Proprietary",
+                "Context Size": "16k",
+                "BLEU": "X",
+                "ROUGE": "X",
+                "ChrF": "X",
+                "BERTScore": "X",
+                "BERTScore (Normalized)": "X",
+                "Submitted By": "🏟 Long Code Arena Team",
+            },
+            {
+                "Model Name": "CodeLlama-7b (instruct)",
+                "Availability": "Llama 2 license",
+                "Context Size": "16k",
+                "BLEU": "X",
+                "ROUGE": "X",
+                "ChrF": "X",
+                "BERTScore": "X",
+                "BERTScore (Normalized)": "X",
+                "Submitted By": "🏟 Long Code Arena Team",
+            },
+        ]
+    )
+    return stub_df

src/submission_uploader.py

@@ -0,0 +1,146 @@
+import json
+import logging
+import os
+from typing import List, Optional
+
+from huggingface_hub import (CommitInfo, CommitOperationAdd, Discussion, HfApi,
+                             HfFileSystem)
+
+from .tasks import TASKS_PRETTY_REVERSE
+
+
+class AlreadyExists(Exception):
+    pass
+
+
+class SubmissionUploader:
+    """Class for adding new files to a dataset on a Hub and opening a PR.
+
+    Heavily influenced by these amazing spaces:
+    * https://huggingface.co/spaces/safetensors/convert
+    * https://huggingface.co/spaces/gaia-benchmark/leaderboard
+    """
+
+    def __init__(self, dataset_id: str):
+        self._api = HfApi(token=os.environ["HF_TOKEN"])
+        self._fs = HfFileSystem(token=os.environ["HF_TOKEN"])
+        self._dataset_id = dataset_id
+
+    def _get_previous_pr(self, pr_title: str) -> Optional[Discussion]:
+        """Searches among discussions of dataset repo for a PR with the given title."""
+        try:
+            discussions = self._api.get_repo_discussions(repo_id=self._dataset_id)
+        except Exception:
+            return None
+        for discussion in discussions:
+            if (
+                discussion.status == "open"
+                and discussion.is_pull_request
+                and discussion.title == pr_title
+            ):
+                return discussion
+
+    def _upload_files(
+        self,
+        task_id: str,
+        model_folder: str,
+        model_name_pretty: str,
+        model_availability: str,
+        urls: str,
+        context_size: str,
+        submitted_by: str,
+        filenames: Optional[List[str]],
+    ) -> List[CommitOperationAdd]:
+        # add predictions files
+        commit_operations = [
+            CommitOperationAdd(
+                path_in_repo=f"{task_id}/{model_folder}/predictions/{filename}",
+                path_or_fileobj=filename,
+            )
+            for filename in filenames
+        ]
+
+        # add metadata file
+        metadata_dict = {
+            "model_name": model_name_pretty,
+            "model_availability": model_availability,
+            "urls": urls,
+            "context_size": context_size,
+            "submitted_by": submitted_by,
+        }
+        with open("metadata.json", "w") as f:
+            json.dump(metadata_dict, f)
+        commit_operations.append(
+            CommitOperationAdd(
+                path_in_repo=f"{task_id}/predictions/metadata.json",
+                path_or_fileobj="metadata.json",
+            )
+        )
+
+        return commit_operations
+
+    def upload_files(
+        self,
+        task_pretty: str,
+        model_folder: str,
+        model_name_pretty: str,
+        model_availability: str,
+        urls: str,
+        context_size: str,
+        submitted_by: str,
+        filenames: Optional[List[str]],
+        force: bool = False,
+    ) -> Optional[CommitInfo]:
+        pr_title = f"🚀 New submission to {task_pretty} task: {model_name_pretty} with {context_size} context size from {submitted_by}"
+
+        task_id = TASKS_PRETTY_REVERSE[task_pretty]
+
+        if not force:
+            if model_name_pretty in self._fs.ls(
+                f"{self._dataset_id}/{task_id}/predictions"
+            ) and all(
+                filename
+                in self._fs.ls(
+                    f"{self._dataset_id}/{task_id}/predictions/{model_name_pretty}"
+                )
+                for filename in filenames + ["metadata.json"]
+            ):
+                raise AlreadyExists(
+                    f"{model_name_pretty} is already present in {self._dataset_id}."
+                )
+
+            prev_pr = self._get_previous_pr(pr_title)
+            if prev_pr is not None:
+                url = f"https://huggingface.co/{self._dataset_id}/discussions/{prev_pr.num}"
+                raise AlreadyExists(
+                    f"{self._dataset_id} already has an open PR for this submission: {url}."
+                )
+
+        commit_operations = self._upload_files(
+            task_id=task_id,
+            model_folder=model_folder,
+            model_name_pretty=model_name_pretty,
+            model_availability=model_availability,
+            urls=urls,
+            context_size=context_size,
+            submitted_by=submitted_by,
+            filenames=filenames,
+        )
+
+        new_pr = self._api.create_commit(
+            repo_id=self._dataset_id,
+            operations=commit_operations,
+            commit_message=pr_title,
+            commit_description=f"""New submission to {task_pretty} task in 🏟️Long Code Arena benchmark!
+                
+                * Model name: {model_name_pretty}
+                * Model availability: {model_availability}
+                * Context Size: {context_size}
+                * Relevant URLs: {urls}
+                * Submitted By: {submitted_by}
+                """,
+            create_pr=True,
+        )
+        logging.info(f"PR created at {new_pr.pr_url}")
+
+        return new_pr

src/tasks.py

@@ -0,0 +1,29 @@
+TASKS_PRETTY = {
+    "cmg": "Commit Message Generation",
+    "bug_localization": "Bug Localization on Issue",
+    "module_to_text": "Module-to-Text",
+    "library_usage": "Library Usage Examples Generation",
+    "project_code_completion": "Project-level Code Completion",
+    "bug_localization_build_logs": "Bug Localization on Build Logs",
+}
+TASKS_PRETTY_REVERSE = {value: key for key, value in TASKS_PRETTY.items()}
+
+TASKS_DESCRIPTIONS = {
+    "Commit Message Generation": """# Commit Message Generation\n
+        
+        Our Commit Message Generation benchmark 🤗 [JetBrains-Research/lca-cmg](https://huggingface.co/datasets/JetBrains-Research/lca-cmg) includes 163 manually curated commits from Python projects.  
+        
+        We use the following metrics for evaluation:
+        * [BLEU](https://huggingface.co/spaces/evaluate-metric/sacrebleu)
+        * [ROUGE](https://huggingface.co/spaces/evaluate-metric/rouge)
+        * [ChrF](https://huggingface.co/spaces/evaluate-metric/chrf)
+        * [BERTScore](https://huggingface.co/spaces/evaluate-metric/berscore)
+        
+        For further details on the dataset and the baselines from 🏟️ Long Code Arena Team, refer to `commit_message_generation` folder in [our baselines repository](https://github.com/JetBrains-Research/lca-baselines) or to our preprint (TODO).       
+        """,
+    "Bug Localization on Issue": "cool description for Bug Localization on Issue task",
+    "Module-to-Text": "cool description for Module-to-Text task",
+    "Library Usage Examples Generation": "cool description for Library Usage Examples Generation task",
+    "Project-level Code Completion": "cool description for Project-level Code Completion task",
+    "Bug Localization on Build Logs": "cool description for Bug Localization on Build Logs task",
+}