Shaltiel commited on
Commit
98eee75
1 Parent(s): ffccb41

Added admin exception for counters

Browse files
Files changed (1) hide show
  1. main_backend_lighteval.py +38 -36
main_backend_lighteval.py CHANGED
@@ -81,44 +81,47 @@ def run_auto_eval():
81
  pp.pprint(dict(message="Number of params too big, can't run this model", params=eval_request.params))
82
  return
83
 
84
- counter_key = f'count_{instance_size}_{instance_type}'
85
- if not counter_key in requests_seen:
86
- requests_seen[counter_key] = 0
87
- if requests_seen[counter_key] >= cap:
88
- set_eval_request(
89
- api=API,
90
- eval_request=eval_request,
91
- set_to_status=REJECTED_STATUS,
92
- hf_repo=QUEUE_REPO,
93
- local_dir=EVAL_REQUESTS_PATH_BACKEND,
94
- )
95
- pp.pprint(dict(message="Reached maximum cap for requests of this instance type this month", counter=counter_key, instance_type=instance_type, cap=cap))
96
- return
97
-
98
- # next, check to see who made the last commit to this repo - keep track of that. One person shouldn't commit more
99
- # than 4 models in one month.
100
- user = eval_request.user_info['name']
101
- if user in requests_seen and len(requests_seen[user]) >= 4:
102
- set_eval_request(
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  api=API,
104
- eval_request=eval_request,
105
- set_to_status=REJECTED_STATUS,
106
  hf_repo=QUEUE_REPO,
107
- local_dir=EVAL_REQUESTS_PATH_BACKEND,
108
  )
109
- pp.pprint(dict(message="Reached maximum cap for requests for this user this month", counter=counter_key, user=user))
110
- return
111
- if not user in requests_seen:
112
- requests_seen[user] = []
113
-
114
- requests_seen[user].append(dict(model_id=eval_request.model, revision=eval_request.revision))
115
- requests_seen[counter_key] += 1
116
- set_requests_seen(
117
- api=API,
118
- requests_seen=requests_seen,
119
- hf_repo=QUEUE_REPO,
120
- local_dir=EVAL_REQUESTS_PATH_BACKEND
121
- )
122
 
123
  set_eval_request(
124
  api=API,
@@ -127,7 +130,6 @@ def run_auto_eval():
127
  hf_repo=QUEUE_REPO,
128
  local_dir=EVAL_REQUESTS_PATH_BACKEND,
129
  )
130
-
131
 
132
 
133
  run_evaluation(
 
81
  pp.pprint(dict(message="Number of params too big, can't run this model", params=eval_request.params))
82
  return
83
 
84
+ # ignore counters, from admins
85
+ is_admin = any(org['name'] == 'hebrew-llm-leaderboard' for org in eval_request.user_info.get('orgs', []))
86
+ if not is_admin:
87
+ counter_key = f'count_{instance_size}_{instance_type}'
88
+ if not counter_key in requests_seen:
89
+ requests_seen[counter_key] = 0
90
+ if requests_seen[counter_key] >= cap:
91
+ set_eval_request(
92
+ api=API,
93
+ eval_request=eval_request,
94
+ set_to_status=REJECTED_STATUS,
95
+ hf_repo=QUEUE_REPO,
96
+ local_dir=EVAL_REQUESTS_PATH_BACKEND,
97
+ )
98
+ pp.pprint(dict(message="Reached maximum cap for requests of this instance type this month", counter=counter_key, instance_type=instance_type, cap=cap))
99
+ return
100
+
101
+ # next, check the user that submitted the request - allow up to 4 per user
102
+ user = eval_request.user_info['name']
103
+ if user in requests_seen and len(requests_seen[user]) >= 4:
104
+ set_eval_request(
105
+ api=API,
106
+ eval_request=eval_request,
107
+ set_to_status=REJECTED_STATUS,
108
+ hf_repo=QUEUE_REPO,
109
+ local_dir=EVAL_REQUESTS_PATH_BACKEND,
110
+ )
111
+ pp.pprint(dict(message="Reached maximum cap for requests for this user this month", counter=counter_key, user=user))
112
+ return
113
+ if not user in requests_seen:
114
+ requests_seen[user] = []
115
+
116
+ requests_seen[user].append(dict(model_id=eval_request.model, revision=eval_request.revision))
117
+ requests_seen[counter_key] += 1
118
+ set_requests_seen(
119
  api=API,
120
+ requests_seen=requests_seen,
 
121
  hf_repo=QUEUE_REPO,
122
+ local_dir=EVAL_REQUESTS_PATH_BACKEND
123
  )
124
+ # end of counters check for non-admins
 
 
 
 
 
 
 
 
 
 
 
 
125
 
126
  set_eval_request(
127
  api=API,
 
130
  hf_repo=QUEUE_REPO,
131
  local_dir=EVAL_REQUESTS_PATH_BACKEND,
132
  )
 
133
 
134
 
135
  run_evaluation(