clefourrier HF staff commited on
Commit
2671d62
1 Parent(s): 2e485f8

submission-fix (#803)

Browse files

- fix submission check logic (e6d10f6c0c8d5d5bdc367a0fbf4f2545bb8d64d1)
- add line break for doc sentence (35a2b072c7ad323cb2dc4af7446cae596006a181)
- fix tests and add new function to utils.py (e641f846e40711431d2eb3cb2163f7573f09fe35)
- update gitignore (13df6817e1929893cb4f38667951e88fd625538c)

.gitignore CHANGED
@@ -10,10 +10,13 @@ __pycache__/
10
  .python-version
11
  .profile_app.python
12
  *pstats
13
- poetry.lock
14
 
15
  eval-queue/
16
  eval-results/
17
  dynamic-info/
 
 
 
18
 
19
  src/assets/model_counts.html
 
10
  .python-version
11
  .profile_app.python
12
  *pstats
13
+ *.lock
14
 
15
  eval-queue/
16
  eval-results/
17
  dynamic-info/
18
+ downloads/
19
+ model-votes/
20
+ open-llm-leaderboard___contents/
21
 
22
  src/assets/model_counts.html
src/display/about.py CHANGED
@@ -4,7 +4,7 @@ TITLE = """<h1 style="text-align:left;float:left; id="space-title">🤗 Open LLM
4
 
5
  INTRODUCTION_TEXT = """
6
  Feeling lost? Documentation is [here](https://huggingface.co/docs/leaderboards/open_llm_leaderboard/about) 📄
7
- You'll notably find explanations on the evaluations we are using, reproducibility guidelines, best practices on how to submit a model, and our FAQ.
8
  """
9
 
10
  icons = f"""
 
4
 
5
  INTRODUCTION_TEXT = """
6
  Feeling lost? Documentation is [here](https://huggingface.co/docs/leaderboards/open_llm_leaderboard/about) 📄
7
+ \n You'll notably find explanations on the evaluations we are using, reproducibility guidelines, best practices on how to submit a model, and our FAQ.
8
  """
9
 
10
  icons = f"""
src/display/utils.py CHANGED
@@ -9,6 +9,11 @@ import pandas as pd
9
  # Configure logging
10
  logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
11
 
 
 
 
 
 
12
 
13
  def parse_datetime(datetime_str):
14
  formats = [
 
9
  # Configure logging
10
  logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
11
 
12
+ # Convert ISO 8601 dates to datetime objects for comparison
13
+ def parse_iso8601_datetime(date_str):
14
+ if date_str.endswith('Z'):
15
+ date_str = date_str[:-1] + '+00:00'
16
+ return datetime.fromisoformat(date_str)
17
 
18
  def parse_datetime(datetime_str):
19
  formats = [
src/submission/check_validity.py CHANGED
@@ -9,6 +9,7 @@ from huggingface_hub import ModelCard
9
  from huggingface_hub.hf_api import ModelInfo, get_safetensors_metadata
10
  from transformers import AutoConfig, AutoTokenizer
11
 
 
12
  from src.envs import HAS_HIGHER_RATE_LIMIT
13
 
14
 
@@ -103,25 +104,29 @@ def get_model_arch(model_info: ModelInfo):
103
 
104
 
105
  def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota):
 
 
 
 
106
  if org_or_user not in users_to_submission_dates:
107
  return True, ""
 
108
  submission_dates = sorted(users_to_submission_dates[org_or_user])
 
109
 
110
- time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
111
- submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
 
 
112
 
113
  num_models_submitted_in_period = len(submissions_after_timelimit)
114
- if org_or_user in HAS_HIGHER_RATE_LIMIT:
115
- rate_limit_quota = 2 * rate_limit_quota
116
-
117
- if num_models_submitted_in_period > rate_limit_quota:
118
- error_msg = f"Organisation or user `{org_or_user}`"
119
- error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
120
- error_msg += f"in the last {rate_limit_period} days.\n"
121
- error_msg += (
122
- "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
123
- )
124
  return False, error_msg
 
125
  return True, ""
126
 
127
 
 
9
  from huggingface_hub.hf_api import ModelInfo, get_safetensors_metadata
10
  from transformers import AutoConfig, AutoTokenizer
11
 
12
+ from src.display.utils import parse_iso8601_datetime
13
  from src.envs import HAS_HIGHER_RATE_LIMIT
14
 
15
 
 
104
 
105
 
106
  def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota):
107
+ # Increase quota first if user has higher limits
108
+ if org_or_user in HAS_HIGHER_RATE_LIMIT:
109
+ rate_limit_quota *= 2
110
+
111
  if org_or_user not in users_to_submission_dates:
112
  return True, ""
113
+
114
  submission_dates = sorted(users_to_submission_dates[org_or_user])
115
+ time_limit = datetime.now(timezone.utc) - timedelta(days=rate_limit_period)
116
 
117
+ submissions_after_timelimit = [
118
+ parse_iso8601_datetime(d) for d in submission_dates
119
+ if parse_iso8601_datetime(d) > time_limit
120
+ ]
121
 
122
  num_models_submitted_in_period = len(submissions_after_timelimit)
123
+
124
+ # Use >= to correctly enforce the rate limit
125
+ if num_models_submitted_in_period >= rate_limit_quota:
126
+ error_msg = f"Organisation or user `{org_or_user}` already has {num_models_submitted_in_period} model requests submitted in the last {rate_limit_period} days.\n"
127
+ error_msg += "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
 
 
 
 
 
128
  return False, error_msg
129
+
130
  return True, ""
131
 
132
 
src/submission/submit.py CHANGED
@@ -63,11 +63,6 @@ def add_new_eval(
63
  if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS:
64
  return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
65
 
66
- if model == "CohereForAI/c4ai-command-r-plus":
67
- return styled_warning(
68
- "This model cannot be submitted manually on the leaderboard before the transformers release."
69
- )
70
-
71
  # Does the model actually exist?
72
  if revision == "":
73
  revision = "main"
 
63
  if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS:
64
  return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
65
 
 
 
 
 
 
66
  # Does the model actually exist?
67
  if revision == "":
68
  revision = "main"
tests/submission/test_user_submission_permission.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import unittest
2
+ from unittest.mock import patch
3
+ from datetime import datetime, timedelta, timezone
4
+
5
+ from src.submission.check_validity import user_submission_permission
6
+ from src.envs import RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
7
+
8
+ class TestUserSubmissionPermission(unittest.TestCase):
9
+
10
+ def setUp(self):
11
+ self.user_name = "test_user"
12
+ self.rate_limit_period = RATE_LIMIT_PERIOD
13
+ self.rate_limit_quota = RATE_LIMIT_QUOTA
14
+ self.fixed_now = datetime(2023, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
15
+ # Submission dates that simulate various test cases
16
+ self.users_to_submission_dates = {
17
+ "test_user": [
18
+ (self.fixed_now - timedelta(days=1)).isoformat(),
19
+ (self.fixed_now - timedelta(days=2)).isoformat(),
20
+ (self.fixed_now - timedelta(days=3)).isoformat(),
21
+ (self.fixed_now - timedelta(days=4)).isoformat(),
22
+ ]
23
+ }
24
+
25
+ @staticmethod
26
+ def fixed_datetime_now(tz=None):
27
+ return datetime(2023, 6, 1, 12, 0, 0, tzinfo=timezone.utc)
28
+
29
+ @patch('src.submission.check_validity.datetime')
30
+ def test_user_below_quota(self, mock_datetime):
31
+ mock_datetime.now.side_effect = self.fixed_datetime_now
32
+ mock_datetime.fromisoformat = datetime.fromisoformat
33
+ allowed, message = user_submission_permission(
34
+ self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
35
+ )
36
+ self.assertTrue(allowed)
37
+
38
+ @patch('src.submission.check_validity.datetime')
39
+ def test_user_at_quota(self, mock_datetime):
40
+ mock_datetime.now.side_effect = self.fixed_datetime_now
41
+ mock_datetime.fromisoformat = datetime.fromisoformat
42
+
43
+ # Add one more submission to reach the quota
44
+ self.users_to_submission_dates["test_user"].append(self.fixed_now.isoformat())
45
+
46
+ allowed, message = user_submission_permission(
47
+ self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
48
+ )
49
+ self.assertFalse(allowed)
50
+ expected_message = (
51
+ f"Organisation or user `{self.user_name}` already has {self.rate_limit_quota} model requests submitted "
52
+ f"in the last {self.rate_limit_period} days.\n"
53
+ "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
54
+ )
55
+ self.assertEqual(message, expected_message)
56
+
57
+ @patch('src.submission.check_validity.datetime')
58
+ def test_user_above_quota(self, mock_datetime):
59
+ mock_datetime.now.side_effect = self.fixed_datetime_now
60
+ mock_datetime.fromisoformat = datetime.fromisoformat
61
+ # Add more than quota submissions
62
+ for _ in range(self.rate_limit_quota + 1):
63
+ self.users_to_submission_dates["test_user"].append(self.fixed_now.isoformat())
64
+ allowed, message = user_submission_permission(
65
+ self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
66
+ )
67
+ self.assertFalse(allowed)
68
+
69
+ def test_user_no_previous_submissions(self):
70
+ allowed, message = user_submission_permission(
71
+ "new_user", self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
72
+ )
73
+ self.assertTrue(allowed)
74
+
75
+ @patch('src.submission.check_validity.HAS_HIGHER_RATE_LIMIT', ["specific_user"])
76
+ @patch('src.submission.check_validity.datetime')
77
+ def test_user_higher_rate_limit(self, mock_datetime):
78
+ mock_datetime.now.side_effect = self.fixed_datetime_now
79
+ mock_datetime.fromisoformat = datetime.fromisoformat
80
+ self.users_to_submission_dates["specific_user"] = [self.fixed_now.isoformat()] * (self.rate_limit_quota + 1)
81
+ allowed, message = user_submission_permission(
82
+ "specific_user", self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
83
+ )
84
+ self.assertTrue(allowed)
85
+
86
+ @patch('src.submission.check_validity.datetime')
87
+ def test_submission_just_outside_window(self, mock_datetime):
88
+ mock_datetime.now.side_effect = self.fixed_datetime_now
89
+ mock_datetime.fromisoformat = datetime.fromisoformat
90
+ old_submission = (self.fixed_now - timedelta(days=self.rate_limit_period, seconds=1)).isoformat()
91
+ self.users_to_submission_dates["test_user"] = [old_submission]
92
+ allowed, message = user_submission_permission(
93
+ self.user_name, self.users_to_submission_dates, self.rate_limit_period, self.rate_limit_quota
94
+ )
95
+ self.assertTrue(allowed)
96
+
97
+ if __name__ == '__main__':
98
+ unittest.main()