Added rate limiting system to the leaderboard to prevent abuse

app.py

@@ -26,6 +26,7 @@ from src.display_models.utils import (
     styled_warning,
 )
 from src.load_from_hub import get_evaluation_queue_df, get_leaderboard_df, is_model_on_hub, load_all_info_from_hub
+from src.rate_limiting import user_submission_permission
 
 pd.set_option("display.precision", 1)
 
@@ -52,6 +53,9 @@ api = HfApi(token=H4_TOKEN)
 def restart_space():
     api.restart_space(repo_id="HuggingFaceH4/open_llm_leaderboard", token=H4_TOKEN)
 
+# Rate limit variables
+RATE_LIMIT_PERIOD = 7
+RATE_LIMIT_QUOTA = 5
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
@@ -77,12 +81,12 @@ BENCHMARK_COLS = [
 ]
 
 ## LOAD INFO FROM HUB
-eval_queue, requested_models, eval_results = load_all_info_from_hub(
+eval_queue, requested_models, eval_results, users_to_submission_dates = load_all_info_from_hub(
     QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH
 )
 
 if not IS_PUBLIC:
-    (eval_queue_private, requested_models_private, eval_results_private,) = load_all_info_from_hub(
+    (eval_queue_private, requested_models_private, eval_results_private, _) = load_all_info_from_hub(
         PRIVATE_QUEUE_REPO,
         PRIVATE_RESULTS_REPO,
         EVAL_REQUESTS_PATH_PRIVATE,
@@ -122,6 +126,14 @@ def add_new_eval(
     precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
+    num_models_submitted_in_period = user_submission_permission(model, users_to_submission_dates, RATE_LIMIT_PERIOD)
+    if num_models_submitted_in_period > RATE_LIMIT_QUOTA:
+        error_msg = f"Organisation or user `{model.split('/')[0]}`"
+        error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
+        error_msg += f"in the last {RATE_LIMIT_PERIOD} days.\n"
+        error_msg += "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
+        return styled_error(error_msg)
+
     if model_type is None or model_type == "":
         return styled_error("Please select a model type.")
 

src/load_from_hub.py

@@ -4,6 +4,7 @@ import os
 import pandas as pd
 from huggingface_hub import Repository
 from transformers import AutoConfig
+from collections import defaultdict
 
 from src.assets.hardcoded_evals import baseline, gpt4_values, gpt35_values
 from src.display_models.get_model_metadata import apply_metadata
@@ -16,6 +17,7 @@ IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
 def get_all_requested_models(requested_models_dir: str) -> set[str]:
     depth = 1
     file_names = []
+    users_to_submission_dates = defaultdict(list)
 
     for root, _, files in os.walk(requested_models_dir):
         current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
@@ -26,7 +28,13 @@ def get_all_requested_models(requested_models_dir: str) -> set[str]:
                     info = json.load(f)
                     file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
 
-    return set(file_names)
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates
 
 
 def load_all_info_from_hub(QUEUE_REPO: str, RESULTS_REPO: str, QUEUE_PATH: str, RESULTS_PATH: str) -> list[Repository]:
@@ -50,9 +58,9 @@ def load_all_info_from_hub(QUEUE_REPO: str, RESULTS_REPO: str, QUEUE_PATH: str,
     )
     eval_results_repo.git_pull()
 
-    requested_models = get_all_requested_models("eval-queue")
+    requested_models, users_to_submission_dates = get_all_requested_models("eval-queue")
 
-    return eval_queue_repo, requested_models, eval_results_repo
+    return eval_queue_repo, requested_models, eval_results_repo, users_to_submission_dates
 
 
 def get_leaderboard_df(

src/rate_limiting.py

@@ -0,0 +1,16 @@
+
+from datetime import datetime, timezone, timedelta
+
+
+def user_submission_permission(submission_name, users_to_submission_dates, rate_limit_period):
+    org_or_user, _ = submission_name.split("/")
+    if org_or_user not in users_to_submission_dates:
+        return 0
+    submission_dates = sorted(users_to_submission_dates[org_or_user])
+
+    time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
+
+    return len(submissions_after_timelimit)
+
+