Spaces:
Paused
Paused
Added admin exception for counters
Browse files- main_backend_lighteval.py +38 -36
main_backend_lighteval.py
CHANGED
@@ -81,44 +81,47 @@ def run_auto_eval():
|
|
81 |
pp.pprint(dict(message="Number of params too big, can't run this model", params=eval_request.params))
|
82 |
return
|
83 |
|
84 |
-
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
103 |
api=API,
|
104 |
-
|
105 |
-
set_to_status=REJECTED_STATUS,
|
106 |
hf_repo=QUEUE_REPO,
|
107 |
-
local_dir=EVAL_REQUESTS_PATH_BACKEND
|
108 |
)
|
109 |
-
|
110 |
-
return
|
111 |
-
if not user in requests_seen:
|
112 |
-
requests_seen[user] = []
|
113 |
-
|
114 |
-
requests_seen[user].append(dict(model_id=eval_request.model, revision=eval_request.revision))
|
115 |
-
requests_seen[counter_key] += 1
|
116 |
-
set_requests_seen(
|
117 |
-
api=API,
|
118 |
-
requests_seen=requests_seen,
|
119 |
-
hf_repo=QUEUE_REPO,
|
120 |
-
local_dir=EVAL_REQUESTS_PATH_BACKEND
|
121 |
-
)
|
122 |
|
123 |
set_eval_request(
|
124 |
api=API,
|
@@ -127,7 +130,6 @@ def run_auto_eval():
|
|
127 |
hf_repo=QUEUE_REPO,
|
128 |
local_dir=EVAL_REQUESTS_PATH_BACKEND,
|
129 |
)
|
130 |
-
|
131 |
|
132 |
|
133 |
run_evaluation(
|
|
|
81 |
pp.pprint(dict(message="Number of params too big, can't run this model", params=eval_request.params))
|
82 |
return
|
83 |
|
84 |
+
# ignore counters, from admins
|
85 |
+
is_admin = any(org['name'] == 'hebrew-llm-leaderboard' for org in eval_request.user_info.get('orgs', []))
|
86 |
+
if not is_admin:
|
87 |
+
counter_key = f'count_{instance_size}_{instance_type}'
|
88 |
+
if not counter_key in requests_seen:
|
89 |
+
requests_seen[counter_key] = 0
|
90 |
+
if requests_seen[counter_key] >= cap:
|
91 |
+
set_eval_request(
|
92 |
+
api=API,
|
93 |
+
eval_request=eval_request,
|
94 |
+
set_to_status=REJECTED_STATUS,
|
95 |
+
hf_repo=QUEUE_REPO,
|
96 |
+
local_dir=EVAL_REQUESTS_PATH_BACKEND,
|
97 |
+
)
|
98 |
+
pp.pprint(dict(message="Reached maximum cap for requests of this instance type this month", counter=counter_key, instance_type=instance_type, cap=cap))
|
99 |
+
return
|
100 |
+
|
101 |
+
# next, check the user that submitted the request - allow up to 4 per user
|
102 |
+
user = eval_request.user_info['name']
|
103 |
+
if user in requests_seen and len(requests_seen[user]) >= 4:
|
104 |
+
set_eval_request(
|
105 |
+
api=API,
|
106 |
+
eval_request=eval_request,
|
107 |
+
set_to_status=REJECTED_STATUS,
|
108 |
+
hf_repo=QUEUE_REPO,
|
109 |
+
local_dir=EVAL_REQUESTS_PATH_BACKEND,
|
110 |
+
)
|
111 |
+
pp.pprint(dict(message="Reached maximum cap for requests for this user this month", counter=counter_key, user=user))
|
112 |
+
return
|
113 |
+
if not user in requests_seen:
|
114 |
+
requests_seen[user] = []
|
115 |
+
|
116 |
+
requests_seen[user].append(dict(model_id=eval_request.model, revision=eval_request.revision))
|
117 |
+
requests_seen[counter_key] += 1
|
118 |
+
set_requests_seen(
|
119 |
api=API,
|
120 |
+
requests_seen=requests_seen,
|
|
|
121 |
hf_repo=QUEUE_REPO,
|
122 |
+
local_dir=EVAL_REQUESTS_PATH_BACKEND
|
123 |
)
|
124 |
+
# end of counters check for non-admins
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
125 |
|
126 |
set_eval_request(
|
127 |
api=API,
|
|
|
130 |
hf_repo=QUEUE_REPO,
|
131 |
local_dir=EVAL_REQUESTS_PATH_BACKEND,
|
132 |
)
|
|
|
133 |
|
134 |
|
135 |
run_evaluation(
|