initial attempt to make leaderboard working

app.py

@@ -1,13 +1,9 @@
 import gradio as gr
 from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
 
 from src.about import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
     TITLE,
@@ -16,46 +12,23 @@ from src.display.css_html_js import custom_css
 from src.display.utils import (
     BENCHMARK_COLS,
     COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
     AutoEvalColumn,
-    ModelType,
     fields,
-    WeightType,
-    Precision
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
+from src.populate import get_leaderboard_df
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
 
+LEADERBOARD_DF = get_leaderboard_df(
+    EVAL_RESULTS_PATH + "/ARFBench_leaderboard.csv", EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS
+)
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
@@ -68,20 +41,15 @@ def init_leaderboard(dataframe):
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        search_columns=[AutoEvalColumn.model.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
             ColumnFilter(
-                AutoEvalColumn.params.name,
+                AutoEvalColumn.pass_at_1.name,
                 type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+                min=0,
+                max=100,
+                label="pass@1 score",
             ),
         ],
         bool_checkboxgroup_label="Hide models",
@@ -95,99 +63,12 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 ARFBench Leaderboard", elem_id="arfbench-tab-table", id=0):
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+        with gr.TabItem("📝 About", elem_id="about-tab-table", id=1):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
-
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(
@@ -198,7 +79,7 @@ with demo:
                 show_copy_button=True,
             )
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+scheduler = None
+demo.queue(default_concurrency_limit=40)
+if __name__ == "__main__":
+    demo.launch()

src/about.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from enum import Enum
 
+
 @dataclass
 class Task:
     benchmark: str
@@ -11,13 +12,13 @@ class Task:
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
     task0 = Task("anli_r1", "acc", "ANLI")
     task1 = Task("logiqa", "acc_norm", "LogiQA")
 
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
 
+NUM_FEWSHOT = 0  # Change with your few shot
+# ---------------------------------------------------
 
 
 # Your leaderboard name
@@ -25,8 +26,20 @@ TITLE = """<h1 align="center" id="space-title">ARFBench Multimodal Time Series R
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-**ARF**Bench (**A**nomaly **R**easoning **F**ramework Benchmark) is a multimodal time-series reasoning benchmark consisting of 550 question-answer (QA) pairs composed from
-real-world incident data collected at Datadog, a leading observability platform.
+**ARF**Bench (**A**nomaly **R**easoning **F**ramework Benchmark) is a
+multimodal time-series reasoning benchmark consisting of 550 question-answer
+(QA) pairs composed from real-world incident data collected at Datadog,
+a leading observability platform.
+
+The benchmark evaluates models across various aspects of time-series anomaly reasoning:
+- **Presence**: Detecting if anomalies exist in the data
+- **Identification**: Identifying specific anomalous metrics
+- **Start Time**: Determining when anomalies began
+- **End Time**: Determining when anomalies ended
+- **Magnitude**: Assessing the severity of anomalies
+- **Categorization**: Classifying anomaly types
+- **Correlation**: Understanding relationships between anomalies
+- **Indicator**: Identifying leading indicators
 """
 
 # Which evaluations are you running? how can people reproduce what you have?

src/display/formatting.py

@@ -1,4 +1,6 @@
 def model_hyperlink(link, model_name):
+    if model_name == "":
+        return model_name
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
 

src/display/utils.py

@@ -1,9 +1,6 @@
 from dataclasses import dataclass, make_dataclass
 from enum import Enum
 
-import pandas as pd
-
-from src.about import Tasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -20,29 +17,28 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
 
-## Leaderboard columns
+
+# ARFBench Leaderboard columns
 auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+# Model column (always displayed)
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-#auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-#for task in Tasks:
-#    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-#auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-#auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-#auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-#auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-#auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-#auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-#auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-#auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
+# Performance metrics
+auto_eval_column_dict.append(["pass_at_1", ColumnContent, ColumnContent("pass@1", "number", True)])
+auto_eval_column_dict.append(["pass_at_5", ColumnContent, ColumnContent("pass@5", "number", True)])
+# Specific benchmark metrics
+auto_eval_column_dict.append(["presence", ColumnContent, ColumnContent("Presence", "number", True)])
+auto_eval_column_dict.append(["identification", ColumnContent, ColumnContent("Identification", "number", True)])
+auto_eval_column_dict.append(["start_time", ColumnContent, ColumnContent("Start Time", "number", True)])
+auto_eval_column_dict.append(["end_time", ColumnContent, ColumnContent("End Time", "number", True)])
+auto_eval_column_dict.append(["magnitude", ColumnContent, ColumnContent("Magnitude", "number", True)])
+auto_eval_column_dict.append(["categorization", ColumnContent, ColumnContent("Categorization", "number", True)])
+auto_eval_column_dict.append(["correlation", ColumnContent, ColumnContent("Correlation", "number", True)])
+auto_eval_column_dict.append(["indicator", ColumnContent, ColumnContent("Indicator", "number", True)])
+
+# We use make dataclass to dynamically fill the scores
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -53,12 +49,13 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
+
 ## All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
     display_name: str = ""
-    symbol: str = "" # emoji
+    symbol: str = ""  # emoji
 
 
 class ModelType(Enum):
@@ -83,11 +80,13 @@ class ModelType(Enum):
             return ModelType.TSFM
         return ModelType.Unknown
 
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
 
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
@@ -100,11 +99,23 @@ class Precision(Enum):
             return Precision.bfloat16
         return Precision.Unknown
 
+
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-
+# Define the benchmark columns for ARFBench
+BENCHMARK_COLS = [
+    "pass_at_1",
+    "pass_at_5",
+    "presence",
+    "identification",
+    "start_time",
+    "end_time",
+    "magnitude",
+    "categorization",
+    "correlation",
+    "indicator",
+]

src/envs.py

@@ -4,9 +4,11 @@ from huggingface_hub import HfApi
 
 # Info to change for your repository
 # ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
 
-OWNER = "Datadog" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = (
+    "Datadog"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
+)
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/ARFBench"
@@ -14,7 +16,7 @@ QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO = f"{OWNER}/results"
 
 # If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
+CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")

src/populate.py

@@ -1,58 +1,63 @@
-import json
 import os
-
 import pandas as pd
 
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
+from src.display.formatting import make_clickable_model
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
+    """Creates a dataframe from the static CSV file"""
+    # Read the static CSV file
+    csv_path = os.path.join("results", "ARFBench_leaderboard.csv")
+
+    if not os.path.exists(csv_path):
+        raise FileNotFoundError(f"CSV file not found at {csv_path}")
+
+    df = pd.read_csv(csv_path)
+
+    # Clean up column names to match our schema
+    df.columns = df.columns.str.replace("pass@1", "pass_at_1")
+    df.columns = df.columns.str.replace("pass@5", "pass_at_5")
+    df.columns = df.columns.str.replace("Start Time", "start_time")
+    df.columns = df.columns.str.replace("End Time", "end_time")
+    df.columns = df.columns.str.lower()
+
+    # Make model names clickable
+    df["model"] = df["model"].apply(make_clickable_model)
+
+    # Sort by pass@1 performance (descending)
+    df = df.sort_values(by=["pass_at_1"], ascending=False)
+
+    # Round numeric columns to 2 decimal places
+    numeric_cols = [
+        "pass_at_1",
+        "pass_at_5",
+        "presence",
+        "identification",
+        "start_time",
+        "end_time",
+        "magnitude",
+        "categorization",
+        "correlation",
+        "indicator",
+    ]
+
+    for col in numeric_cols:
+        if col in df.columns:
+            df[col] = df[col].round(2)
+
+    # Handle missing values - replace with 0 or appropriate value
+    df = df.fillna(0)
+
+    # Select only the columns we need
+    available_cols = [col for col in cols if col in df.columns]
+    df = df[available_cols]
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
     return df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]
+    """Creates empty dataframes for evaluation queues since we're using
+    static data"""
+    # Return empty dataframes for the queue system
+    empty_df = pd.DataFrame(columns=cols)
+    return empty_df, empty_df, empty_df