DamonDemon commited on
Commit
308f73c
1 Parent(s): 6244676
.gitignore ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ .env
4
+ .ipynb_checkpoints
5
+ *.pyc
6
+ *ipynb
7
+ .vscode/
8
+ .idea/
README.md CHANGED
@@ -1,13 +1,12 @@
1
  ---
2
  title: UnlearnCanvas Benchmark
3
- emoji: 🐨
4
- colorFrom: green
5
- colorTo: yellow
6
  sdk: gradio
7
- sdk_version: 4.19.1
8
  app_file: app.py
9
  pinned: false
10
- license: apache-2.0
11
  ---
12
 
13
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
  title: UnlearnCanvas Benchmark
3
+ emoji: 🎨
4
+ colorFrom: red
5
+ colorTo: purple
6
  sdk: gradio
7
+ sdk_version: 4.17.0
8
  app_file: app.py
9
  pinned: false
 
10
  ---
11
 
12
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,501 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import pandas as pd
3
+ from src.display.about import (
4
+ CITATION_BUTTON_LABEL,
5
+ CITATION_BUTTON_TEXT,
6
+ EVALUATION_QUEUE_TEXT,
7
+ INTRODUCTION_TEXT,
8
+ LLM_BENCHMARKS_TEXT,
9
+ FAQ_TEXT,
10
+ TITLE,
11
+ )
12
+ from src.display.css_html_js import custom_css
13
+ from src.display.utils import (
14
+ BENCHMARK_COLS,
15
+ COLS,
16
+ EVAL_COLS,
17
+ EVAL_TYPES,
18
+ NUMERIC_INTERVALS,
19
+ TYPES,
20
+ AutoEvalColumn,
21
+ ModelType,
22
+ fields,
23
+ WeightType,
24
+ Precision
25
+ )
26
+ from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
27
+ from PIL import Image
28
+ # from src.populate import get_evaluation_queue_df, get_leaderboard_df
29
+ # from src.submission.submit import add_new_eval
30
+ # from src.tools.collections import update_collections
31
+ # from src.tools.plots import (
32
+ # create_metric_plot_obj,
33
+ # create_plot_df,
34
+ # create_scores_df,
35
+ # )
36
+ from dummydatagen import dummy_data_for_plot, create_metric_plot_obj_1, dummydf
37
+ import copy
38
+
39
+
40
+ def restart_space():
41
+ API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
42
+
43
+
44
+ def add_average_col(df):
45
+
46
+ always_here_cols = [
47
+ "Model", "Agent", "Opponent Model", "Opponent Agent"
48
+ ]
49
+ desired_col = [i for i in list(df.columns) if i not in always_here_cols]
50
+ newdf = df[desired_col].mean(axis=1).round(3)
51
+ return newdf
52
+
53
+
54
+ gtbench_raw_data = dummydf()
55
+ gtbench_raw_data["Average"] = add_average_col(gtbench_raw_data)
56
+
57
+ column_to_move = "Average"
58
+ # Move the column to the desired index
59
+ gtbench_raw_data.insert(
60
+ 4, column_to_move, gtbench_raw_data.pop(column_to_move))
61
+
62
+ models = list(set(gtbench_raw_data['Model']))
63
+
64
+ opponent_models = list(set(gtbench_raw_data['Opponent Model']))
65
+
66
+
67
+ agents = list(set(gtbench_raw_data['Agent']))
68
+
69
+
70
+ opponent_agents = list(set(gtbench_raw_data['Opponent Agent']))
71
+
72
+ # Searching and filtering
73
+
74
+
75
+ def update_table(
76
+ hidden_df: pd.DataFrame,
77
+ columns: list,
78
+ model1: list,
79
+ model2: list,
80
+ agent1: list,
81
+ agent2: list
82
+ ):
83
+
84
+ filtered_df = select_columns(hidden_df, columns)
85
+
86
+ filtered_df = filter_model1(filtered_df, model1)
87
+ filtered_df = filter_model2(filtered_df, model2)
88
+ filtered_df = filter_agent1(filtered_df, agent1)
89
+ filtered_df = filter_agent2(filtered_df, agent2)
90
+
91
+ return filtered_df
92
+
93
+ # triggered only once at startup => read query parameter if it exists
94
+
95
+
96
+ def load_query(request: gr.Request):
97
+ query = request.query_params.get("query") or ""
98
+ return query, query # return one for the "search_bar", one for a hidden component that triggers a reload only if value has changed
99
+
100
+
101
+ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
102
+ return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
103
+
104
+
105
+ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
106
+ always_here_cols = [
107
+ "Model", "Agent", "Opponent Model", "Opponent Agent"
108
+ ]
109
+ # We use COLS to maintain sorting
110
+ all_columns = games
111
+
112
+ if len(columns) == 0:
113
+ filtered_df = df[
114
+ always_here_cols +
115
+ [c for c in all_columns if c in df.columns]
116
+ ]
117
+ filtered_df["Average"] = add_average_col(filtered_df)
118
+ column_to_move = "Average"
119
+ current_index = filtered_df.columns.get_loc(column_to_move)
120
+
121
+ # Move the column to the desired index
122
+ filtered_df.insert(4, column_to_move, filtered_df.pop(column_to_move))
123
+ return filtered_df
124
+
125
+ filtered_df = df[
126
+ always_here_cols +
127
+ [c for c in all_columns if c in df.columns and c in columns]
128
+ ]
129
+ if "Average" in columns:
130
+ filtered_df["Average"] = add_average_col(filtered_df)
131
+ # Get the current index of the column
132
+ column_to_move = "Average"
133
+ current_index = filtered_df.columns.get_loc(column_to_move)
134
+
135
+ # Move the column to the desired index
136
+ filtered_df.insert(4, column_to_move, filtered_df.pop(column_to_move))
137
+ else:
138
+ if "Average" in filtered_df.columns:
139
+ # Remove the column
140
+ filtered_df = filtered_df.drop(columns=["Average"])
141
+
142
+ return filtered_df
143
+
144
+
145
+ def filter_model1(
146
+ df: pd.DataFrame, model_query: list
147
+ ) -> pd.DataFrame:
148
+ # Show all models
149
+ if len(model_query) == 0:
150
+ return df
151
+ filtered_df = df
152
+
153
+ filtered_df = filtered_df[filtered_df["Model"].isin(
154
+ model_query)]
155
+ return filtered_df
156
+
157
+
158
+ def filter_model2(
159
+ df: pd.DataFrame, model_query: list
160
+ ) -> pd.DataFrame:
161
+ # Show all models
162
+ if len(model_query) == 0:
163
+ return df
164
+ filtered_df = df
165
+
166
+ filtered_df = filtered_df[filtered_df["Opponent Model"].isin(
167
+ model_query)]
168
+ return filtered_df
169
+
170
+
171
+ def filter_agent1(
172
+ df: pd.DataFrame, agent_query: list
173
+ ) -> pd.DataFrame:
174
+ # Show all models
175
+ if len(agent_query) == 0:
176
+ return df
177
+ filtered_df = df
178
+
179
+ filtered_df = filtered_df[filtered_df["Agent"].isin(
180
+ agent_query)]
181
+ return filtered_df
182
+
183
+
184
+ def filter_agent2(
185
+ df: pd.DataFrame, agent_query: list
186
+ ) -> pd.DataFrame:
187
+ # Show all models
188
+ if len(agent_query) == 0:
189
+ return df
190
+ filtered_df = df
191
+
192
+ filtered_df = filtered_df[filtered_df["Opponent Agent"].isin(
193
+ agent_query)]
194
+ return filtered_df
195
+
196
+
197
+ # leaderboard_df = filter_models(leaderboard_df, [t.to_str(" : ") for t in ModelType], list(NUMERIC_INTERVALS.keys()), [i.value.name for i in Precision], False, False)
198
+
199
+
200
+ class LLM_Model:
201
+ def __init__(self, t_value, model_value, average_value, arc_value, hellaSwag_value, mmlu_value) -> None:
202
+ self.t = t_value
203
+ self.model = model_value
204
+ self.average = average_value
205
+ self.arc = arc_value
206
+ self.hellaSwag = hellaSwag_value
207
+ self.mmlu = mmlu_value
208
+
209
+
210
+ games = ["Breakthrough", "Connect Four", "Blind Auction", "Kuhn Poker",
211
+ "Liar's Dice", "Negotiation", "Nim", "Pig", "Iterated Prisoner's Dilemma", "Tic-Tac-Toe"]
212
+
213
+ # models = ["gpt-35-turbo-1106", "gpt-4", "Llama-2-70b-chat-hf", "CodeLlama-34b-Instruct-hf",
214
+ # "CodeLlama-70b-Instruct-hf", "Mistral-7B-Instruct-v01", "Mistral-7B-OpenOrca"]
215
+
216
+ # agents = ["Prompt Agent", "CoT Agent", "SC-CoT Agent",
217
+ # "ToT Agent", "MCTS", "Random", "TitforTat"]
218
+
219
+ demo = gr.Blocks(css=custom_css)
220
+
221
+
222
+ def load_image(image_path):
223
+ image = Image.open(image_path)
224
+ return image
225
+
226
+
227
+ with demo:
228
+ with gr.Row():
229
+ gr.Image("./assets/logo.png", height="200px", width="200px", scale=0.1,
230
+ show_download_button=False, container=False)
231
+ gr.HTML(TITLE, elem_id="title")
232
+
233
+ gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
234
+ with gr.Tabs(elem_classes="tab-buttons") as tabs:
235
+ with gr.TabItem("🏅 GTBench", elem_id="llm-benchmark-tab-table", id=0):
236
+ with gr.Row():
237
+ with gr.Column():
238
+ with gr.Row():
239
+
240
+ shown_columns = gr.CheckboxGroup(
241
+ choices=[
242
+ 'Average'
243
+ ]+games,
244
+ label="Select columns to show",
245
+ elem_id="column-select",
246
+ interactive=True,
247
+ )
248
+ with gr.Column(min_width=320):
249
+ # with gr.Box(elem_id="box-filter"):
250
+ model1_column = gr.CheckboxGroup(
251
+ label="Model",
252
+ choices=models,
253
+ interactive=True,
254
+ elem_id="filter-columns-type",
255
+ )
256
+
257
+ agent1_column = gr.CheckboxGroup(
258
+ label="Agents",
259
+ choices=agents,
260
+ interactive=True,
261
+ elem_id="filter-columns-precision",
262
+ )
263
+
264
+ model2_column = gr.CheckboxGroup(
265
+ label="Opponent Model",
266
+ choices=opponent_models,
267
+ interactive=True,
268
+ elem_id="filter-columns-type",
269
+ )
270
+ agent2_column = gr.CheckboxGroup(
271
+ label="Opponent Agents",
272
+ choices=opponent_agents,
273
+ interactive=True,
274
+ elem_id="filter-columns-precision",
275
+ )
276
+ # filter_columns_size = gr.CheckboxGroup(
277
+ # label="Model sizes (in billions of parameters)",
278
+ # choices=[f'NUMERIC_INTERVALS{i}' for i in range(0, 5)],
279
+ # value=[f'NUMERIC_INTERVALS{i}' for i in range(0, 5)],
280
+ # interactive=True,
281
+ # elem_id="filter-columns-size",
282
+ # )
283
+
284
+ leaderboard_table = gr.components.Dataframe(
285
+ value=gtbench_raw_data,
286
+ elem_id="leaderboard-table",
287
+ interactive=False,
288
+ visible=True,
289
+ # column_widths=["2%", "33%"]
290
+ )
291
+
292
+ game_bench_df_for_search = gr.components.Dataframe(
293
+ value=gtbench_raw_data,
294
+ elem_id="leaderboard-table",
295
+ interactive=False,
296
+ visible=False,
297
+ # column_widths=["2%", "33%"]
298
+ )
299
+
300
+ # Dummy leaderboard for handling the case when the user uses backspace key
301
+ # hidden_leaderboard_table_for_search = gr.components.Dataframe(
302
+ # value=[],
303
+ # headers=COLS,
304
+ # datatype=TYPES,
305
+ # visible=False,
306
+ # )
307
+ # search_bar.submit(
308
+ # update_table,
309
+ # [
310
+ # # hidden_leaderboard_table_for_search,
311
+ # # shown_columns,
312
+ # # filter_columns_type,
313
+ # # filter_columns_precision,
314
+ # # filter_columns_size,
315
+ # # deleted_models_visibility,
316
+ # # flagged_models_visibility,
317
+ # # search_bar,
318
+ # ],
319
+ # leaderboard_table,
320
+ # )
321
+
322
+ # # Define a hidden component that will trigger a reload only if a query parameter has be set
323
+ # hidden_search_bar = gr.Textbox(value="", visible=False)
324
+ # hidden_search_bar.change(
325
+ # update_table,
326
+ # [
327
+ # hidden_leaderboard_table_for_search,
328
+ # shown_columns,
329
+ # filter_columns_type,
330
+ # filter_columns_precision,
331
+ # filter_columns_size,
332
+ # deleted_models_visibility,
333
+ # flagged_models_visibility,
334
+ # search_bar,
335
+ # ],
336
+ # leaderboard_table,
337
+ # )
338
+ # # Check query parameter once at startup and update search bar + hidden component
339
+ # demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
340
+
341
+ for selector in [shown_columns, model1_column, model2_column, agent1_column, agent2_column]:
342
+ selector.change(
343
+ update_table,
344
+ [
345
+ game_bench_df_for_search,
346
+ shown_columns,
347
+ model1_column,
348
+ model2_column,
349
+ agent1_column,
350
+ agent2_column
351
+ # filter_columns_precision,
352
+ # None, # filter_columns_size,
353
+ # None, # deleted_models_visibility,
354
+ # None, # flagged_models_visibility,
355
+ # None, # search_bar,
356
+ ],
357
+ leaderboard_table,
358
+ queue=True,
359
+ )
360
+
361
+ # with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=4):
362
+ # with gr.Row():
363
+ # with gr.Column():
364
+ # chart = create_metric_plot_obj_1(
365
+ # dummy_data_for_plot(
366
+ # ["Metric1", "Metric2", 'Metric3']),
367
+ # ["Metric1", "Metric2", "Metric3"],
368
+ # title="Average of Top Scores and Human Baseline Over Time (from last update)",
369
+ # )
370
+ # gr.Plot(value=chart, min_width=500)
371
+
372
+ with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
373
+ gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
374
+ gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")
375
+
376
+ '''
377
+ with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
378
+ with gr.Column():
379
+ with gr.Row():
380
+ gr.Markdown(EVALUATION_QUEUE_TEXT,
381
+ elem_classes="markdown-text")
382
+
383
+ with gr.Column():
384
+ with gr.Accordion(
385
+ f"✅ Finished Evaluations ({9})",
386
+ open=False,
387
+ ):
388
+ with gr.Row():
389
+ finished_eval_table = gr.components.Dataframe(
390
+ value=None,
391
+ headers=EVAL_COLS,
392
+ datatype=EVAL_TYPES,
393
+ row_count=5,
394
+ )
395
+ with gr.Accordion(
396
+ f"🔄 Running Evaluation Queue ({5})",
397
+ open=False,
398
+ ):
399
+ with gr.Row():
400
+ running_eval_table = gr.components.Dataframe(
401
+ value=None,
402
+ headers=EVAL_COLS,
403
+ datatype=EVAL_TYPES,
404
+ row_count=5,
405
+ )
406
+
407
+ with gr.Accordion(
408
+ f"⏳ Pending Evaluation Queue ({7})",
409
+ open=False,
410
+ ):
411
+ with gr.Row():
412
+ pending_eval_table = gr.components.Dataframe(
413
+ value=None,
414
+ headers=EVAL_COLS,
415
+ datatype=EVAL_TYPES,
416
+ row_count=5,
417
+ )
418
+ with gr.Row():
419
+ gr.Markdown("# ✉️✨ Submit your Agent here!",
420
+ elem_classes="markdown-text")
421
+
422
+ with gr.Row():
423
+ with gr.Column():
424
+ model_name_textbox = gr.Textbox(label="Agent name")
425
+ # revision_name_textbox = gr.Textbox(
426
+ # label="Revision commit", placeholder="main")
427
+ # private = gr.Checkbox(
428
+ # False, label="Private", visible=not IS_PUBLIC)
429
+ model_type = gr.Dropdown(
430
+ choices=[t.to_str(" : ")
431
+ for t in ModelType if t != ModelType.Unknown],
432
+ label="Agent type",
433
+ multiselect=False,
434
+ value=ModelType.FT.to_str(" : "),
435
+ interactive=True,
436
+ )
437
+
438
+ # with gr.Column():
439
+ # precision = gr.Dropdown(
440
+ # choices=[i.value.name for i in Precision if i !=
441
+ # Precision.Unknown],
442
+ # label="Precision",
443
+ # multiselect=False,
444
+ # value="float16",
445
+ # interactive=True,
446
+ # )
447
+ # weight_type = gr.Dropdown(
448
+ # choices=[i.value.name for i in WeightType],
449
+ # label="Weights type",
450
+ # multiselect=False,
451
+ # value="Original",
452
+ # interactive=True,
453
+ # )
454
+ # base_model_name_textbox = gr.Textbox(
455
+ # label="Base model (for delta or adapter weights)")
456
+
457
+ submit_button = gr.Button("Submit Eval")
458
+ submission_result = gr.Markdown()
459
+ # submit_button.click(
460
+ # add_new_eval,
461
+ # [
462
+ # model_name_textbox,
463
+ # base_model_name_textbox,
464
+ # revision_name_textbox,
465
+ # precision,
466
+ # private,
467
+ # weight_type,
468
+ # model_type,
469
+ # ],
470
+ # submission_result,
471
+ # )
472
+
473
+ '''
474
+ with gr.Row():
475
+ with gr.Accordion("📙 Citation", open=False):
476
+ citation_button = gr.Textbox(
477
+ value=CITATION_BUTTON_TEXT,
478
+ label=CITATION_BUTTON_LABEL,
479
+ lines=20,
480
+ elem_id="citation-button",
481
+ show_copy_button=True,
482
+ )
483
+
484
+ # scheduler = BackgroundScheduler()
485
+ # scheduler.add_job(restart_space, "interval", seconds=1800)
486
+ # scheduler.start()
487
+ demo.launch()
488
+ # Both launches the space and its CI
489
+ # configure_space_ci(
490
+ # demo.queue(default_concurrency_limit=40),
491
+ # trusted_authors=[], # add manually trusted authors
492
+ # private="True", # ephemeral spaces will have same visibility as the main space. Otherwise, set to `True` or `False` explicitly.
493
+ # variables={}, # We overwrite HF_HOME as tmp CI spaces will have no cache
494
+ # secrets=["HF_TOKEN", "H4_TOKEN"], # which secret do I want to copy from the main space? Can be a `List[str]`."HF_TOKEN", "H4_TOKEN"
495
+ # hardware=None, # "cpu-basic" by default. Otherwise set to "auto" to have same hardware as the main space or any valid string value.
496
+ # storage=None, # no storage by default. Otherwise set to "auto" to have same storage as the main space or any valid string value.
497
+ # ).launch()
498
+
499
+
500
+ # notes: opponent model , opponent agent
501
+ # column is games
assets/gtbench_results.csv ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Model,Agent,Opponent Model,Opponent Agent,Tic-Tac-Toe,Connect Four,Breakthrough,Liar's Dice,Blind Auction,Negotiation,Kuhn Poker,Nim,Pig,Iterated Prisoner's Dilemma,
2
+ GPT-3.5-turbo,Prompt,GPT-3.5-turbo-1106,prompt agent,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000
3
+ GPT-4,Prompt,GPT-3.5-turbo-1106,prompt agent,-0.111,0.080,0.320,0.800,0.040,-0.281,0.400,0.080,-0.040,0.004,0.129
4
+ GPT-4,CoT,GPT-3.5-turbo-1106,prompt agent,-0.022,-0.080,0.560,0.240,0.069,0.135,0.440,0.040,0.040,-0.160,0.126
5
+ GPT-3.5-turbo,CoT,GPT-3.5-turbo-1106,prompt agent,0.277,-0.320,-0.120,0.440,0.115,-0.207,0.120,-0.040,-0.160,0.126,0.023
6
+ GPT-3.5-turbo,SC-CoT,GPT-3.5-turbo-1106,prompt agent,0.409,-0.040,-0.160,0.520,-0.120,-0.315,-0.080,0.000,-0.080,-0.155,-0.002
7
+ GPT-3.5-turbo,ToT,GPT-3.5-turbo-1106,prompt agent,-0.045,0.240,0.160,0.000,-0.120,0.183,0.000,0.120,-0.400,-0.191,-0.005
8
+ Codellama-34b-instruct,Prompt,GPT-3.5-turbo-1106,prompt agent,0.333,-0.100,-0.800,-0.400,-0.250,0.216,-0.160,0.360,0.120,0.600,-0.008
9
+ Llama-2-70b-chat,SC-CoT,GPT-3.5-turbo-1106,prompt agent,-0.469,-0.160,-0.680,0.160,-0.040,0.052,0.120,0.040,0.040,0.296,-0.064
10
+ Codellama-34b-instruct,CoT,GPT-3.5-turbo-1106,prompt agent,0.316,-0.360,-0.760,-0.320,-0.268,0.085,0.000,0.480,-0.080,0.032,-0.088
11
+ Llama-2-70b-chat,CoT,GPT-3.5-turbo-1106,prompt agent,-0.500,0.080,-0.800,0.265,-0.086,0.128,-0.200,0.061,-0.160,0.324,-0.089
12
+ Mistral-7b-Orca,CoT,GPT-3.5-turbo-1106,prompt agent,-0.077,-0.120,-0.320,-0.560,0.133,0.078,0.000,0.360,-0.680,0.055,-0.113
13
+ Codellama-34b-instruct,SC-CoT,GPT-3.5-turbo-1106,prompt agent,0.122,-0.600,-0.560,-0.280,-0.348,0.095,0.000,0.160,0.120,0.008,-0.128
14
+ Mistral-7b-Orca,SC-CoT,GPT-3.5-turbo-1106,prompt agent,-0.200,-0.080,-0.400,-0.640,0.082,0.364,-0.040,0.440,-0.840,0.013,-0.130
15
+ Codellama-34b-instruct,ToT,GPT-3.5-turbo-1106,prompt agent,-0.021,-0.160,-0.600,-0.520,-0.304,0.098,0.000,-0.040,-0.160,0.237,-0.147
16
+ Llama-2-70b-chat,Prompt,GPT-3.5-turbo-1106,prompt agent,-0.366,-1.000,-0.440,-0.160,-0.075,-0.033,-0.040,0.800,-0.020,-0.712,-0.205
17
+ Mistral-7b-Orca,ToT,GPT-3.5-turbo-1106,prompt agent,-0.179,-0.800,-0.320,-0.440,-0.047,0.299,-0.200,-0.080,-0.840,0.162,-0.245
18
+ Mistral-7b-Orca,Prompt,GPT-3.5-turbo-1106,prompt agent,-0.429,-0.840,-0.680,-0.680,-0.069,-0.114,-0.040,-0.080,0.000,-0.182,-0.311
19
+ GPT-4,Prompt,GPT-4,prompt agent,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000
20
+ Codellama-34b-instruct,Prompt,GPT-4,prompt agent,-0.064,0.720,-0.600,-0.640,-0.148,0.000,0.080,0.160,0.040,0.342,-0.011
21
+ Codellama-34b-instruct,CoT,GPT-4,prompt agent,0.022,0.560,-1.000,-0.800,0.449,-0.078,0.080,0.200,-0.080,0.224,-0.042
22
+ Llama-2-70b-chat,Prompt,GPT-4,prompt agent,-0.938,0.960,-0.920,-0.720,-0.250,0.000,-0.040,0.360,0.200,0.333,-0.101
23
+ Llama-2-70b-chat,CoT,GPT-4,prompt agent,-0.286,0.200,-0.880,-0.917,-0.417,0.201,0.000,-0.026,-0.360,0.173,-0.231
dummydatagen.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ from datetime import datetime, timedelta
3
+ import numpy as np
4
+ import pandas as pd
5
+ import plotly.express as px
6
+ from plotly.graph_objs import Figure
7
+
8
+ # Dummy data creation
9
+
10
+
11
+ def dummy_data_for_plot(metrics, num_days=30):
12
+ dates = [datetime.now() - timedelta(days=i) for i in range(num_days)]
13
+ data = []
14
+
15
+ for metric in metrics:
16
+ for date in dates:
17
+ model = f"Model_{metric}"
18
+ score = np.random.uniform(50, 55)
19
+ data.append([date, metric, score, model])
20
+
21
+ df = pd.DataFrame(data, columns=["date", "task", "score", "model"])
22
+ return df
23
+
24
+
25
+ def create_metric_plot_obj_1(
26
+ df: pd.DataFrame, metrics: list[str], title: str
27
+ ) -> Figure:
28
+ """
29
+ Create a Plotly figure object with lines representing different metrics
30
+ and horizontal dotted lines representing human baselines.
31
+
32
+ :param df: The DataFrame containing the metric values, names, and dates.
33
+ :param metrics: A list of strings representing the names of the metrics
34
+ to be included in the plot.
35
+ :param title: A string representing the title of the plot.
36
+ :return: A Plotly figure object with lines representing metrics and
37
+ horizontal dotted lines representing human baselines.
38
+ """
39
+
40
+ # Filter the DataFrame based on the specified metrics
41
+ df = df[df["task"].isin(metrics)]
42
+
43
+ # Filter the human baselines based on the specified metrics
44
+ # filtered_human_baselines = {k: v for k, v in HUMAN_BASELINE.items() if k in metrics}
45
+
46
+ # Create a line figure using plotly express with specified markers and custom data
47
+ fig = px.line(
48
+ df,
49
+ x="date",
50
+ y="score",
51
+ color="task",
52
+ markers=True,
53
+ custom_data=["task", "score", "model"],
54
+ title=title,
55
+ )
56
+
57
+ # Update hovertemplate for better hover interaction experience
58
+ fig.update_traces(
59
+ hovertemplate="<br>".join(
60
+ [
61
+ "Model Name: %{customdata[2]}",
62
+ "Metric Name: %{customdata[0]}",
63
+ "Date: %{x}",
64
+ "Metric Value: %{y}",
65
+ ]
66
+ )
67
+ )
68
+
69
+ # Update the range of the y-axis
70
+ fig.update_layout(yaxis_range=[0, 100])
71
+
72
+ # Create a dictionary to hold the color mapping for each metric
73
+ metric_color_mapping = {}
74
+
75
+ # Map each metric name to its color in the figure
76
+ for trace in fig.data:
77
+ metric_color_mapping[trace.name] = trace.line.color
78
+
79
+ # Iterate over filtered human baselines and add horizontal lines to the figure
80
+ # for metric, value in filtered_human_baselines.items():
81
+ # color = metric_color_mapping.get(metric, "blue") # Retrieve color from mapping; default to blue if not found
82
+ # location = "top left" if metric == "HellaSwag" else "bottom left" # Set annotation position
83
+ # # Add horizontal line with matched color and positioned annotation
84
+ # fig.add_hline(
85
+ # y=value,
86
+ # line_dash="dot",
87
+ # annotation_text=f"{metric} human baseline",
88
+ # annotation_position=location,
89
+ # annotation_font_size=10,
90
+ # annotation_font_color=color,
91
+ # line_color=color,
92
+ # )
93
+
94
+ return fig
95
+
96
+
97
+ def dummydf():
98
+ # data = [{"Model": "gpt-35-turbo-1106",
99
+ # "Agent": "prompt agent",
100
+ # "Opponent Model": "gpt-4",
101
+ # "Opponent Agent": "prompt agent",
102
+ # 'Breakthrough': 0,
103
+ # 'Connect Four': 0,
104
+ # 'Blind Auction': 0,
105
+ # 'Kuhn Poker': 0,
106
+ # "Liar's Dice": 0,
107
+ # 'Negotiation': 0,
108
+ # 'Nim': 0,
109
+ # 'Pig': 0,
110
+ # 'Iterated Prisoners Dilemma': 0,
111
+ # 'Tic-Tac-Toe': 0
112
+ # },
113
+ # {"Model": "Llama-2-70b-chat-hf",
114
+ # "Agent": "prompt agent",
115
+ # "Opponent Model": "gpt-4",
116
+ # "Opponent Agent": "prompt agent",
117
+ # 'Breakthrough': 1,
118
+ # 'Connect Four': 0,
119
+ # 'Blind Auction': 0,
120
+ # 'Kuhn Poker': 0,
121
+ # "Liar's Dice": 0,
122
+ # 'Negotiation': 0,
123
+ # 'Nim': 0,
124
+ # 'Pig': 0,
125
+ # 'Iterated Prisoners Dilemma': 0,
126
+ # 'Tic-Tac-Toe': 0
127
+ # },
128
+ # {"Model": "gpt-35-turbo-1106",
129
+ # "Agent": "ToT agent",
130
+ # "Opponent Model": "gpt-4",
131
+ # "Opponent Agent": "prompt agent",
132
+ # 'Breakthrough': 0,
133
+ # 'Connect Four': 0,
134
+ # 'Blind Auction': 0,
135
+ # 'Kuhn Poker': 0,
136
+ # "Liar's Dice": 0,
137
+ # 'Negotiation': 0,
138
+ # 'Nim': 0,
139
+ # 'Pig': 0,
140
+ # 'Iterated Prisoners Dilemma': 0,
141
+ # 'Tic-Tac-Toe': 0
142
+ # },
143
+ # {"Model": "Llama-2-70b-chat-hf",
144
+ # "Agent": "CoT agent",
145
+ # "Opponent Model": "gpt-4",
146
+ # "Opponent Agent": "prompt agent",
147
+ # 'Breakthrough': 0,
148
+ # 'Connect Four': 0,
149
+ # 'Blind Auction': 0,
150
+ # 'Kuhn Poker': 0,
151
+ # "Liar's Dice": 0,
152
+ # 'Negotiation': 0,
153
+ # 'Nim': 0,
154
+ # 'Pig': 0,
155
+ # 'Iterated Prisoners Dilemma': 0,
156
+ # 'Tic-Tac-Toe': 0
157
+ # }]
158
+ df = pd.read_csv('./assets/gtbench_results.csv')
159
+ return df
requirements.txt ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ aiofiles==23.2.1
2
+ altair==5.2.0
3
+ annotated-types==0.6.0
4
+ anyio==4.2.0
5
+ attrs==23.2.0
6
+ certifi==2024.2.2
7
+ charset-normalizer==3.3.2
8
+ click==8.1.7
9
+ colorama==0.4.6
10
+ contourpy==1.2.0
11
+ cycler==0.12.1
12
+ exceptiongroup==1.2.0
13
+ fastapi==0.109.2
14
+ ffmpy==0.3.1
15
+ filelock==3.13.1
16
+ fonttools==4.48.1
17
+ fsspec==2024.2.0
18
+ gradio==4.17.0
19
+ gradio_client==0.9.0
20
+ h11==0.14.0
21
+ httpcore==1.0.2
22
+ httpx==0.26.0
23
+ huggingface-hub==0.20.3
24
+ idna==3.6
25
+ importlib-resources==6.1.1
26
+ Jinja2==3.1.3
27
+ jsonschema==4.21.1
28
+ jsonschema-specifications==2023.12.1
29
+ kiwisolver==1.4.5
30
+ markdown-it-py==3.0.0
31
+ MarkupSafe==2.1.5
32
+ matplotlib==3.7.1
33
+ mdurl==0.1.2
34
+ numpy==1.24.2
35
+ orjson==3.9.13
36
+ packaging==23.2
37
+ pandas==2.0.0
38
+ pillow==10.2.0
39
+ plotly==5.18.0
40
+ pydantic==2.6.1
41
+ pydantic_core==2.16.2
42
+ pydub==0.25.1
43
+ Pygments==2.17.2
44
+ pyparsing==3.1.1
45
+ python-dateutil==2.8.2
46
+ python-multipart==0.0.7
47
+ pytz==2024.1
48
+ PyYAML==6.0.1
49
+ referencing==0.33.0
50
+ regex==2023.12.25
51
+ requests==2.28.2
52
+ rich==13.7.0
53
+ rpds-py==0.17.1
54
+ ruff==0.2.1
55
+ safetensors==0.4.2
56
+ semantic-version==2.10.0
57
+ shellingham==1.5.4
58
+ six==1.16.0
59
+ sniffio==1.3.0
60
+ starlette==0.36.3
61
+ tenacity==8.2.3
62
+ tokenizers==0.15.1
63
+ tomlkit==0.12.0
64
+ toolz==0.12.1
65
+ tqdm==4.66.1
66
+ transformers==4.36.0
67
+ typer==0.9.0
68
+ typing_extensions==4.9.0
69
+ tzdata==2023.4
70
+ urllib3==1.26.18
71
+ uvicorn==0.27.0.post1
72
+ websockets==11.0.3
src/display/about.py ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # from src.display.utils import ModelType
2
+
3
+ TITLE = """
4
+ <h1 id="space-title">UNLEARNCANVAS: A Stylized Image Dataset to Benchmark <br> Machine Unlearning for Diffusion Models</h1>"""
5
+
6
+ INTRODUCTION_TEXT = """
7
+
8
+ paper: https://arxiv.org/abs/2402.11846
9
+
10
+ The rapid advancement of diffusion models (DMs) has not only transformed various real- world industries but has also introduced negative societal concerns, including the generation of harmful content, copyright disputes, and the rise of stereotypes and biases. To mitigate these issues, machine unlearning (MU) has emerged as a potential solution, demonstrating its ability to remove undesired generative capabilities of DMs in various applications. However, by examining existing MU evaluation methods, we uncover several key challenges that can result in incomplete, inaccurate, or biased evaluations for MU in DMs.
11
+
12
+ To address them, we enhance the evaluation metrics for MU, including the introduction of an often-overlooked retainability measurement for DMs post-unlearning. Additionally, we introduce UnlearnCanvas, a comprehensive high-resolution stylized image dataset that facilitates us to evaluate the unlearning of artistic painting styles in conjunction with associated image objects.
13
+
14
+ We show that this dataset plays a pivotal role in establishing a standardized and automated evaluation framework for MU techniques on DMs, featuring 7 quantitative metrics to address various aspects of unlearning effectiveness. Through extensive experiments, we benchmark 5 state-of- the-art MU methods, revealing novel insights into their pros and cons, and the underlying unlearning mechanisms. Furthermore, we demonstrate the potential of UnlearnCanvas to benchmark other generative modeling tasks, such as style transfer. The UnlearnCanvas dataset, benchmark, and the codes to reproduce all the results in this work can be found at https://github. com/OPTML-Group/UnlearnCanvas.
15
+
16
+ """
17
+
18
+ LLM_BENCHMARKS_TEXT = f"""
19
+ # Context
20
+
21
+ ## How it works
22
+
23
+ We evaluate LLMs on 10 widely recognized game-theoretic tasks, including
24
+
25
+ - <a href="https://en.wikipedia.org/wiki/Tic-tac-toe" target="_blank"> Tic-Tac-Toe</a>
26
+ - <a href="https://en.wikipedia.org/wiki/Connect_Four" target="_blank"> Connect-4 </a>
27
+ - <a href="https://en.wikipedia.org/wiki/Breakthrough_(board_game)" target="_blank"> Breakthrough</a>
28
+ - <a href="https://en.wikipedia.org/wiki/Nim" target="_blank"> Nim</a>
29
+ - <a href="https://en.wikipedia.org/wiki/First-price_sealed-bid_auction" target="_blank"> Blind Auction</a>
30
+ - <a href="https://en.wikipedia.org/wiki/Kuhn_poker" target="_blank"> Kuhn Poker</a>
31
+ - <a href="https://en.wikipedia.org/wiki/Liar\%27s_dice" target="_blank"> Liar's Dice</a>
32
+ - <a href="https://arxiv.org/pdf/1706.05125.pdf" target="_blank"> Negotiation</a>
33
+ - <a href="https://en.wikipedia.org/wiki/Pig_(dice_game)" target="_blank"> Pig</a>
34
+ - <a href="https://en.wikipedia.org/wiki/Prisoner\%27s_dilemma" target="_blank"> Prisoner's Dilemma</a>
35
+
36
+ ## Metric
37
+ We use Normalized Relative Advantage (NRA) to evaluation the performance of LLM agents. NRA(agent1, agent2) > 0 means agent1 has higher win rate/earn more rewards than the opponent agent2.
38
+
39
+ Please refer to GTBench paper for more detail.
40
+
41
+ ## Takeaways
42
+ - LLM agents failed in the complete-information and deterministic games
43
+ - LLM agents are competitive in the probabilistic games
44
+ - CodePretraining benefits game-theoretic tasks.
45
+ - Advanced Reasoning Methods Do Not Always Help.
46
+
47
+ ## Contact
48
+ Please feel free to contact Jinhao <jd3734@drexel.edu> and Renming <rmzhang@bu.edu> if you have any questions.
49
+
50
+ """
51
+
52
+ FAQ_TEXT = """
53
+ """
54
+
55
+
56
+ EVALUATION_QUEUE_TEXT = """
57
+ # Evaluation for the GTBench leaderboard
58
+
59
+ """
60
+
61
+ CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
62
+
63
+ CITATION_BUTTON_TEXT = r"""
64
+ @article{zhang2024unlearncanvas,
65
+ title={UnlearnCanvas: A Stylized Image Dataset to Benchmark Machine Unlearning for Diffusion Models},
66
+ author={Zhang, Yihua and Zhang, Yimeng and Yao, Yuguang and Jia, Jinghan and Liu, Jiancheng and Liu, Xiaoming and Liu, Sijia},
67
+ journal={arXiv preprint arXiv:2402.11846},
68
+ year={2024}
69
+ }
70
+ """
src/display/css_html_js.py ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ custom_css = """
2
+
3
+ .markdown-text {
4
+ font-size: 16px !important;
5
+ }
6
+
7
+ #models-to-add-text {
8
+ font-size: 18px !important;
9
+ }
10
+
11
+ #citation-button span {
12
+ font-size: 16px !important;
13
+ }
14
+
15
+ #citation-button textarea {
16
+ font-size: 16px !important;
17
+ }
18
+
19
+ #citation-button > label > button {
20
+ margin: 6px;
21
+ transform: scale(1.3);
22
+ }
23
+
24
+ #leaderboard-table {
25
+ margin-top: 15px
26
+ }
27
+
28
+ #leaderboard-table-lite {
29
+ margin-top: 15px
30
+ }
31
+
32
+ #search-bar-table-box > div:first-child {
33
+ background: none;
34
+ border: none;
35
+ }
36
+
37
+ #search-bar {
38
+ padding: 0px;
39
+ }
40
+
41
+
42
+
43
+ /* Limit the width of the first AutoEvalColumn so that names don't expand too much */
44
+ table td:first-child,
45
+ table th:first-child {
46
+ max-width: 400px;
47
+ overflow: auto;
48
+ white-space: nowrap;
49
+ }
50
+
51
+ .tab-buttons button {
52
+ font-size: 20px;
53
+ }
54
+
55
+ #scale-logo {
56
+ border-style: none !important;
57
+ box-shadow: none;
58
+ display: block;
59
+ margin-left: auto;
60
+ margin-right: auto;
61
+ max-width: 600px;
62
+ }
63
+
64
+ #scale-logo .download {
65
+ display: none;
66
+ }
67
+ #filter_type{
68
+ border: 0;
69
+ padding-left: 0;
70
+ padding-top: 0;
71
+ }
72
+ #filter_type label {
73
+ display: flex;
74
+ }
75
+ #filter_type label > span{
76
+ margin-top: var(--spacing-lg);
77
+ margin-right: 0.5em;
78
+ }
79
+ #filter_type label > .wrap{
80
+ width: 103px;
81
+ }
82
+ #filter_type label > .wrap .wrap-inner{
83
+ padding: 2px;
84
+ }
85
+ #filter_type label > .wrap .wrap-inner input{
86
+ width: 1px
87
+ }
88
+ #filter-columns-type{
89
+ border:0;
90
+ padding:0.5;
91
+ }
92
+ #filter-columns-size{
93
+ border:0;
94
+ padding:0.5;
95
+ }
96
+ #box-filter > .form{
97
+ border: 0
98
+ }
99
+ #title{
100
+ margin-top: 110px
101
+ text-align: left;
102
+ display: flex;
103
+ justify-content: flex-start;
104
+ align-items: center;
105
+ }
106
+ """
107
+
108
+ get_window_url_params = """
109
+ function(url_params) {
110
+ const params = new URLSearchParams(window.location.search);
111
+ url_params = Object.fromEntries(params);
112
+ return url_params;
113
+ }
114
+ """
src/display/formatting.py ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from datetime import datetime, timezone
3
+
4
+ from huggingface_hub import HfApi
5
+ from huggingface_hub.hf_api import ModelInfo
6
+
7
+
8
+ API = HfApi()
9
+
10
+ def model_hyperlink(link, model_name):
11
+ return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
12
+
13
+
14
+ def make_clickable_model(model_name):
15
+ link = f"https://huggingface.co/{model_name}"
16
+
17
+ details_model_name = model_name.replace("/", "__")
18
+ details_link = f"https://huggingface.co/datasets/open-llm-leaderboard/details_{details_model_name}"
19
+
20
+ return model_hyperlink(link, model_name) + " " + model_hyperlink(details_link, "📑")
21
+
22
+
23
+ def styled_error(error):
24
+ return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
25
+
26
+
27
+ def styled_warning(warn):
28
+ return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
29
+
30
+
31
+ def styled_message(message):
32
+ return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
33
+
34
+
35
+ def has_no_nan_values(df, columns):
36
+ return df[columns].notna().all(axis=1)
37
+
38
+
39
+ def has_nan_values(df, columns):
40
+ return df[columns].isna().any(axis=1)
src/display/utils.py ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, make_dataclass
2
+ from enum import Enum
3
+
4
+ import pandas as pd
5
+
6
+ def fields(raw_class):
7
+ return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
8
+
9
+
10
+ @dataclass
11
+ class Task:
12
+ benchmark: str
13
+ metric: str
14
+ col_name: str
15
+
16
+ class Tasks(Enum):
17
+ arc = Task("arc:challenge", "acc_norm", "ARC")
18
+ hellaswag = Task("hellaswag", "acc_norm", "HellaSwag")
19
+ mmlu = Task("hendrycksTest", "acc", "MMLU")
20
+ truthfulqa = Task("truthfulqa:mc", "mc2", "TruthfulQA")
21
+ winogrande = Task("winogrande", "acc", "Winogrande")
22
+ gsm8k = Task("gsm8k", "acc", "GSM8K")
23
+
24
+ # These classes are for user facing column names,
25
+ # to avoid having to change them all around the code
26
+ # when a modif is needed
27
+ @dataclass
28
+ class ColumnContent:
29
+ name: str
30
+ type: str
31
+ displayed_by_default: bool
32
+ hidden: bool = False
33
+ never_hidden: bool = False
34
+ dummy: bool = False
35
+
36
+ auto_eval_column_dict = []
37
+ # Init
38
+ auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
39
+ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
40
+ #Scores
41
+ auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
42
+ for task in Tasks:
43
+ auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
44
+ # Model information
45
+ auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
46
+ auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
47
+ auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
48
+ auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
49
+ auto_eval_column_dict.append(["merge", ColumnContent, ColumnContent("Merged", "bool", False)])
50
+ auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
51
+ auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
52
+ auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
53
+ auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
54
+ auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
55
+ auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, False)])
56
+ # Dummy column for the search bar (hidden by the custom CSS)
57
+ auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
58
+
59
+ # We use make dataclass to dynamically fill the scores from Tasks
60
+ AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
61
+
62
+ @dataclass(frozen=True)
63
+ class EvalQueueColumn: # Queue column
64
+ model = ColumnContent("model", "markdown", True)
65
+ revision = ColumnContent("revision", "str", True)
66
+ private = ColumnContent("private", "bool", True)
67
+ precision = ColumnContent("precision", "str", True)
68
+ weight_type = ColumnContent("weight_type", "str", "Original")
69
+ status = ColumnContent("status", "str", True)
70
+
71
+
72
+ baseline_row = {
73
+ AutoEvalColumn.model.name: "<p>Baseline</p>",
74
+ AutoEvalColumn.revision.name: "N/A",
75
+ AutoEvalColumn.precision.name: None,
76
+ AutoEvalColumn.merge.name: False,
77
+ AutoEvalColumn.average.name: 31.0,
78
+ AutoEvalColumn.arc.name: 25.0,
79
+ AutoEvalColumn.hellaswag.name: 25.0,
80
+ AutoEvalColumn.mmlu.name: 25.0,
81
+ AutoEvalColumn.truthfulqa.name: 25.0,
82
+ AutoEvalColumn.winogrande.name: 50.0,
83
+ AutoEvalColumn.gsm8k.name: 0.21,
84
+ AutoEvalColumn.dummy.name: "baseline",
85
+ AutoEvalColumn.model_type.name: "",
86
+ AutoEvalColumn.flagged.name: False,
87
+ }
88
+
89
+ # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)
90
+ # ARC human baseline is 0.80 (source: https://lab42.global/arc/)
91
+ # HellaSwag human baseline is 0.95 (source: https://deepgram.com/learn/hellaswag-llm-benchmark-guide)
92
+ # MMLU human baseline is 0.898 (source: https://openreview.net/forum?id=d7KBjmI3GmQ)
93
+ # TruthfulQA human baseline is 0.94(source: https://arxiv.org/pdf/2109.07958.pdf)
94
+ # Winogrande: https://leaderboard.allenai.org/winogrande/submissions/public
95
+ # GSM8K: paper
96
+ # Define the human baselines
97
+ human_baseline_row = {
98
+ AutoEvalColumn.model.name: "<p>Human performance</p>",
99
+ AutoEvalColumn.revision.name: "N/A",
100
+ AutoEvalColumn.precision.name: None,
101
+ AutoEvalColumn.average.name: 92.75,
102
+ AutoEvalColumn.merge.name: False,
103
+ AutoEvalColumn.arc.name: 80.0,
104
+ AutoEvalColumn.hellaswag.name: 95.0,
105
+ AutoEvalColumn.mmlu.name: 89.8,
106
+ AutoEvalColumn.truthfulqa.name: 94.0,
107
+ AutoEvalColumn.winogrande.name: 94.0,
108
+ AutoEvalColumn.gsm8k.name: 100,
109
+ AutoEvalColumn.dummy.name: "human_baseline",
110
+ AutoEvalColumn.model_type.name: "",
111
+ }
112
+
113
+ @dataclass
114
+ class ModelDetails:
115
+ name: str
116
+ symbol: str = "" # emoji, only for the model type
117
+
118
+
119
+ class ModelType(Enum):
120
+ PT = ModelDetails(name="pretrained", symbol="🟢")
121
+ FT = ModelDetails(name="fine-tuned", symbol="🔶")
122
+ IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
123
+ RL = ModelDetails(name="RL-tuned", symbol="🟦")
124
+ Unknown = ModelDetails(name="", symbol="?")
125
+
126
+ def to_str(self, separator=" "):
127
+ return f"{self.value.symbol}{separator}{self.value.name}"
128
+
129
+ @staticmethod
130
+ def from_str(type):
131
+ if "fine-tuned" in type or "🔶" in type:
132
+ return ModelType.FT
133
+ if "pretrained" in type or "🟢" in type:
134
+ return ModelType.PT
135
+ if "RL-tuned" in type or "🟦" in type:
136
+ return ModelType.RL
137
+ if "instruction-tuned" in type or "⭕" in type:
138
+ return ModelType.IFT
139
+ return ModelType.Unknown
140
+
141
+ class WeightType(Enum):
142
+ Adapter = ModelDetails("Adapter")
143
+ Original = ModelDetails("Original")
144
+ Delta = ModelDetails("Delta")
145
+
146
+ class Precision(Enum):
147
+ float16 = ModelDetails("float16")
148
+ bfloat16 = ModelDetails("bfloat16")
149
+ qt_8bit = ModelDetails("8bit")
150
+ qt_4bit = ModelDetails("4bit")
151
+ qt_GPTQ = ModelDetails("GPTQ")
152
+ Unknown = ModelDetails("?")
153
+
154
+ def from_str(precision):
155
+ if precision in ["torch.float16", "float16"]:
156
+ return Precision.float16
157
+ if precision in ["torch.bfloat16", "bfloat16"]:
158
+ return Precision.bfloat16
159
+ if precision in ["8bit"]:
160
+ return Precision.qt_8bit
161
+ if precision in ["4bit"]:
162
+ return Precision.qt_4bit
163
+ if precision in ["GPTQ", "None"]:
164
+ return Precision.qt_GPTQ
165
+ return Precision.Unknown
166
+
167
+
168
+
169
+
170
+ # Column selection
171
+ COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
172
+ TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
173
+ COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
174
+ TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
175
+
176
+ EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
177
+ EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
178
+
179
+ BENCHMARK_COLS = [t.value.col_name for t in Tasks]
180
+
181
+ NUMERIC_INTERVALS = {
182
+ "?": pd.Interval(-1, 0, closed="right"),
183
+ "~1.5": pd.Interval(0, 2, closed="right"),
184
+ "~3": pd.Interval(2, 4, closed="right"),
185
+ "~7": pd.Interval(4, 9, closed="right"),
186
+ "~13": pd.Interval(9, 20, closed="right"),
187
+ "~35": pd.Interval(20, 45, closed="right"),
188
+ "~60": pd.Interval(45, 70, closed="right"),
189
+ "70+": pd.Interval(70, 10000, closed="right"),
190
+ }
src/envs.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ from huggingface_hub import HfApi
4
+
5
+ # clone / pull the lmeh eval data
6
+ H4_TOKEN = os.environ.get("H4_TOKEN", None)
7
+
8
+ REPO_ID = "HuggingFaceH4/open_llm_leaderboard"
9
+ QUEUE_REPO = "open-llm-leaderboard/requests"
10
+ RESULTS_REPO = "open-llm-leaderboard/results"
11
+
12
+ PRIVATE_QUEUE_REPO = "open-llm-leaderboard/private-requests"
13
+ PRIVATE_RESULTS_REPO = "open-llm-leaderboard/private-results"
14
+
15
+ IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
16
+
17
+ CACHE_PATH=os.getenv("HF_HOME", ".")
18
+
19
+ EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
20
+ EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
21
+
22
+ EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
23
+ EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
24
+
25
+ PATH_TO_COLLECTION = "open-llm-leaderboard/llm-leaderboard-best-models-652d6c7965a4619fb5c27a03"
26
+
27
+ # Rate limit variables
28
+ RATE_LIMIT_PERIOD = 7
29
+ RATE_LIMIT_QUOTA = 5
30
+ HAS_HIGHER_RATE_LIMIT = ["TheBloke"]
31
+
32
+ API = HfApi(token=H4_TOKEN)
src/leaderboard/filter_models.py ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from src.display.formatting import model_hyperlink
2
+ from src.display.utils import AutoEvalColumn
3
+
4
+ # Models which have been flagged by users as being problematic for a reason or another
5
+ # (Model name to forum discussion link)
6
+ FLAGGED_MODELS = {
7
+ "Voicelab/trurl-2-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/202",
8
+ "deepnight-research/llama-2-70B-inst": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/207",
9
+ "Aspik101/trurl-2-13b-pl-instruct_unload": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/213",
10
+ "Fredithefish/ReasonixPajama-3B-HF": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/236",
11
+ "TigerResearch/tigerbot-7b-sft-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/237",
12
+ "gaodrew/gaodrew-gorgonzola-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/215",
13
+ "AIDC-ai-business/Marcoroni-70B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
14
+ "AIDC-ai-business/Marcoroni-13B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
15
+ "AIDC-ai-business/Marcoroni-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
16
+ "fblgit/una-xaberius-34b-v1beta": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/444",
17
+ "jan-hq/trinity-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
18
+ "rwitz2/go-bruins-v2.1.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
19
+ "rwitz2/go-bruins-v2.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
20
+ "GreenNode/GreenNodeLM-v3olet-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
21
+ "GreenNode/GreenNodeLM-7B-v4leo": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
22
+ "GreenNode/LeoScorpius-GreenNode-7B-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
23
+ "viethq188/LeoScorpius-7B-Chat-DPO": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
24
+ "GreenNode/GreenNodeLM-7B-v2leo": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
25
+ "janai-hq/trinity-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
26
+ "ignos/LeoScorpius-GreenNode-Alpaca-7B-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
27
+ "fblgit/una-cybertron-7b-v3-OMA": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
28
+ "mncai/mistral-7b-dpo-merge-v1.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
29
+ "mncai/mistral-7b-dpo-v6": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
30
+ "Toten5/LeoScorpius-GreenNode-7B-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
31
+ "GreenNode/GreenNodeLM-7B-v1olet": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
32
+ "quantumaikr/quantum-dpo-v0.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
33
+ "quantumaikr/quantum-v0.01": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
34
+ "quantumaikr/quantum-trinity-v0.1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
35
+ "mncai/mistral-7b-dpo-v5": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
36
+ "cookinai/BruinHermes": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
37
+ "jan-ai/Pandora-10.7B-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
38
+ "v1olet/v1olet_marcoroni-go-bruins-merge-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
39
+ "v1olet/v1olet_merged_dpo_7B_v3": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
40
+ "rwitz2/pee": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/474",
41
+ }
42
+
43
+ # Models which have been requested by orgs to not be submitted on the leaderboard
44
+ DO_NOT_SUBMIT_MODELS = [
45
+ "Voicelab/trurl-2-13b", # trained on MMLU
46
+ "TigerResearch/tigerbot-70b-chat", # per authors request
47
+ "TigerResearch/tigerbot-70b-chat-v2", # per authors request
48
+ "TigerResearch/tigerbot-70b-chat-v4-4k", # per authors request
49
+ ]
50
+
51
+
52
+ def flag_models(leaderboard_data: list[dict]):
53
+ for model_data in leaderboard_data:
54
+ if model_data["model_name_for_query"] in FLAGGED_MODELS:
55
+ issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
56
+ issue_link = model_hyperlink(
57
+ FLAGGED_MODELS[model_data["model_name_for_query"]],
58
+ f"See discussion #{issue_num}",
59
+ )
60
+ model_data[
61
+ AutoEvalColumn.model.name
62
+ ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
63
+ model_data[AutoEvalColumn.flagged.name] = True
64
+ else:
65
+ model_data[AutoEvalColumn.flagged.name] = False
66
+
67
+
68
+ def remove_forbidden_models(leaderboard_data: list[dict]):
69
+ indices_to_remove = []
70
+ for ix, model in enumerate(leaderboard_data):
71
+ if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
72
+ indices_to_remove.append(ix)
73
+
74
+ for ix in reversed(indices_to_remove):
75
+ leaderboard_data.pop(ix)
76
+ return leaderboard_data
77
+
78
+
79
+ def filter_models(leaderboard_data: list[dict]):
80
+ leaderboard_data = remove_forbidden_models(leaderboard_data)
81
+ flag_models(leaderboard_data)
src/leaderboard/read_evals.py ADDED
@@ -0,0 +1,222 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import glob
2
+ import json
3
+ import math
4
+ import os
5
+ from dataclasses import dataclass
6
+
7
+ import dateutil
8
+ import numpy as np
9
+
10
+ from huggingface_hub import ModelCard
11
+
12
+ from src.display.formatting import make_clickable_model
13
+ from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
14
+ # from src.submission.check_validity import is_model_on_hub
15
+
16
+
17
+ @dataclass
18
+ class EvalResult:
19
+ # Also see src.display.utils.AutoEvalColumn for what will be displayed.
20
+ eval_name: str # org_model_precision (uid)
21
+ full_model: str # org/model (path on hub)
22
+ org: str
23
+ model: str
24
+ revision: str # commit hash, "" if main
25
+ results: dict
26
+ precision: Precision = Precision.Unknown
27
+ model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
28
+ weight_type: WeightType = WeightType.Original # Original or Adapter
29
+ architecture: str = "Unknown" # From config file
30
+ license: str = "?"
31
+ likes: int = 0
32
+ num_params: int = 0
33
+ date: str = "" # submission date of request file
34
+ still_on_hub: bool = False
35
+ merge: bool = False
36
+
37
+ @classmethod
38
+ def init_from_json_file(self, json_filepath):
39
+ """Inits the result from the specific model result file"""
40
+ with open(json_filepath) as fp:
41
+ data = json.load(fp)
42
+
43
+ # We manage the legacy config format
44
+ config = data.get("config", data.get("config_general", None))
45
+
46
+ # Precision
47
+ precision = Precision.from_str(config.get("model_dtype"))
48
+
49
+ # Get model and org
50
+ org_and_model = config.get("model_name", config.get("model_args", None))
51
+ org_and_model = org_and_model.split("/", 1)
52
+
53
+ if len(org_and_model) == 1:
54
+ org = None
55
+ model = org_and_model[0]
56
+ result_key = f"{model}_{precision.value.name}"
57
+ else:
58
+ org = org_and_model[0]
59
+ model = org_and_model[1]
60
+ result_key = f"{org}_{model}_{precision.value.name}"
61
+ full_model = "/".join(org_and_model)
62
+
63
+ try:
64
+ merge = any(t in ["merge", "mergedlm"] for t in ModelCard.load(full_model).data.tags)
65
+ except Exception:
66
+ merge = False
67
+
68
+ still_on_hub, error, model_config = is_model_on_hub(
69
+ full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
70
+ )
71
+ architecture = "?"
72
+ if model_config is not None:
73
+ architectures = getattr(model_config, "architectures", None)
74
+ if architectures:
75
+ architecture = ";".join(architectures)
76
+
77
+ # Extract results available in this file (some results are split in several files)
78
+ results = {}
79
+ for task in Tasks:
80
+ task = task.value
81
+ # We skip old mmlu entries
82
+ wrong_mmlu_version = False
83
+ if task.benchmark == "hendrycksTest":
84
+ for mmlu_k in ["harness|hendrycksTest-abstract_algebra|5", "hendrycksTest-abstract_algebra"]:
85
+ if mmlu_k in data["versions"] and data["versions"][mmlu_k] == 0:
86
+ wrong_mmlu_version = True
87
+
88
+ if wrong_mmlu_version:
89
+ continue
90
+
91
+ # Some truthfulQA values are NaNs
92
+ if task.benchmark == "truthfulqa:mc" and "harness|truthfulqa:mc|0" in data["results"]:
93
+ if math.isnan(float(data["results"]["harness|truthfulqa:mc|0"][task.metric])):
94
+ results[task.benchmark] = 0.0
95
+ continue
96
+
97
+ # We average all scores of a given metric (mostly for mmlu)
98
+ accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
99
+ if accs.size == 0 or any([acc is None for acc in accs]):
100
+ continue
101
+
102
+ mean_acc = np.mean(accs) * 100.0
103
+ results[task.benchmark] = mean_acc
104
+
105
+ return self(
106
+ eval_name=result_key,
107
+ full_model=full_model,
108
+ org=org,
109
+ model=model,
110
+ results=results,
111
+ precision=precision,
112
+ revision= config.get("model_sha", ""),
113
+ still_on_hub=still_on_hub,
114
+ architecture=architecture,
115
+ merge=merge
116
+ )
117
+
118
+ def update_with_request_file(self, requests_path):
119
+ """Finds the relevant request file for the current model and updates info with it"""
120
+ request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
121
+
122
+ try:
123
+ with open(request_file, "r") as f:
124
+ request = json.load(f)
125
+ self.model_type = ModelType.from_str(request.get("model_type", ""))
126
+ self.weight_type = WeightType[request.get("weight_type", "Original")]
127
+ self.license = request.get("license", "?")
128
+ self.likes = request.get("likes", 0)
129
+ self.num_params = request.get("params", 0)
130
+ self.date = request.get("submitted_time", "")
131
+ except Exception:
132
+ print(f"Could not find request file for {self.org}/{self.model}")
133
+
134
+ def to_dict(self):
135
+ """Converts the Eval Result to a dict compatible with our dataframe display"""
136
+ average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
137
+ data_dict = {
138
+ "eval_name": self.eval_name, # not a column, just a save name,
139
+ AutoEvalColumn.precision.name: self.precision.value.name,
140
+ AutoEvalColumn.model_type.name: self.model_type.value.name,
141
+ AutoEvalColumn.merge.name: self.merge,
142
+ AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
143
+ AutoEvalColumn.weight_type.name: self.weight_type.value.name,
144
+ AutoEvalColumn.architecture.name: self.architecture,
145
+ AutoEvalColumn.model.name: make_clickable_model(self.full_model),
146
+ AutoEvalColumn.dummy.name: self.full_model,
147
+ AutoEvalColumn.revision.name: self.revision,
148
+ AutoEvalColumn.average.name: average,
149
+ AutoEvalColumn.license.name: self.license,
150
+ AutoEvalColumn.likes.name: self.likes,
151
+ AutoEvalColumn.params.name: self.num_params,
152
+ AutoEvalColumn.still_on_hub.name: self.still_on_hub,
153
+ }
154
+
155
+ for task in Tasks:
156
+ data_dict[task.value.col_name] = self.results[task.value.benchmark]
157
+
158
+ return data_dict
159
+
160
+
161
+ def get_request_file_for_model(requests_path, model_name, precision):
162
+ """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
163
+ request_files = os.path.join(
164
+ requests_path,
165
+ f"{model_name}_eval_request_*.json",
166
+ )
167
+ request_files = glob.glob(request_files)
168
+
169
+ # Select correct request file (precision)
170
+ request_file = ""
171
+ request_files = sorted(request_files, reverse=True)
172
+ for tmp_request_file in request_files:
173
+ with open(tmp_request_file, "r") as f:
174
+ req_content = json.load(f)
175
+ if (
176
+ req_content["status"] in ["FINISHED"]
177
+ and req_content["precision"] == precision.split(".")[-1]
178
+ ):
179
+ request_file = tmp_request_file
180
+ return request_file
181
+
182
+
183
+ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
184
+ """From the path of the results folder root, extract all needed info for results"""
185
+ model_result_filepaths = []
186
+
187
+ for root, _, files in os.walk(results_path):
188
+ # We should only have json files in model results
189
+ if len(files) == 0 or any([not f.endswith(".json") for f in files]):
190
+ continue
191
+
192
+ # Sort the files by date
193
+ try:
194
+ files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
195
+ except dateutil.parser._parser.ParserError:
196
+ files = [files[-1]]
197
+
198
+ for file in files:
199
+ model_result_filepaths.append(os.path.join(root, file))
200
+
201
+ eval_results = {}
202
+ for model_result_filepath in model_result_filepaths:
203
+ # Creation of result
204
+ eval_result = EvalResult.init_from_json_file(model_result_filepath)
205
+ eval_result.update_with_request_file(requests_path)
206
+
207
+ # Store results of same eval together
208
+ eval_name = eval_result.eval_name
209
+ if eval_name in eval_results.keys():
210
+ eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
211
+ else:
212
+ eval_results[eval_name] = eval_result
213
+
214
+ results = []
215
+ for v in eval_results.values():
216
+ try:
217
+ v.to_dict() # we test if the dict version is complete
218
+ results.append(v)
219
+ except KeyError: # not all eval values present
220
+ continue
221
+
222
+ return results
src/populate.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+
4
+ import pandas as pd
5
+
6
+ from src.display.formatting import has_no_nan_values, make_clickable_model
7
+ from src.display.utils import AutoEvalColumn, EvalQueueColumn, baseline_row
8
+ from src.leaderboard.filter_models import filter_models
9
+ from src.leaderboard.read_evals import get_raw_eval_results
10
+
11
+
12
+ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
13
+ raw_data = get_raw_eval_results(results_path, requests_path)
14
+ all_data_json = [v.to_dict() for v in raw_data]
15
+ all_data_json.append(baseline_row)
16
+ filter_models(all_data_json)
17
+
18
+ df = pd.DataFrame.from_records(all_data_json)
19
+ df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
20
+ df = df[cols].round(decimals=2)
21
+
22
+ # filter out if any of the benchmarks have not been produced
23
+ df = df[has_no_nan_values(df, benchmark_cols)]
24
+ return raw_data, df
25
+
26
+
27
+ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
28
+ entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
29
+ all_evals = []
30
+
31
+ for entry in entries:
32
+ if ".json" in entry:
33
+ file_path = os.path.join(save_path, entry)
34
+ with open(file_path) as fp:
35
+ data = json.load(fp)
36
+
37
+ data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
38
+ data[EvalQueueColumn.revision.name] = data.get("revision", "main")
39
+
40
+ all_evals.append(data)
41
+ elif ".md" not in entry:
42
+ # this is a folder
43
+ sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
44
+ for sub_entry in sub_entries:
45
+ file_path = os.path.join(save_path, entry, sub_entry)
46
+ with open(file_path) as fp:
47
+ data = json.load(fp)
48
+
49
+ data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
50
+ data[EvalQueueColumn.revision.name] = data.get("revision", "main")
51
+ all_evals.append(data)
52
+
53
+ pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
54
+ running_list = [e for e in all_evals if e["status"] == "RUNNING"]
55
+ finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
56
+ df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
57
+ df_running = pd.DataFrame.from_records(running_list, columns=cols)
58
+ df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
59
+ return df_finished[cols], df_running[cols], df_pending[cols]
src/submission/check_validity.py ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import re
4
+ from collections import defaultdict
5
+ from datetime import datetime, timedelta, timezone
6
+
7
+ import huggingface_hub
8
+ from huggingface_hub import ModelCard
9
+ from huggingface_hub.hf_api import ModelInfo
10
+ # from transformers import AutoConfig, AutoTokenizer
11
+ # from transformers.models.auto.tokenization_auto import tokenizer_class_from_name, get_tokenizer_config
12
+
13
+ from src.envs import HAS_HIGHER_RATE_LIMIT
14
+
15
+
16
+ # ht to @Wauplin, thank you for the snippet!
17
+ # See https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/317
18
+ def check_model_card(repo_id: str) -> tuple[bool, str]:
19
+ # Returns operation status, and error message
20
+ try:
21
+ card = ModelCard.load(repo_id)
22
+ except huggingface_hub.utils.EntryNotFoundError:
23
+ return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
24
+
25
+ # Enforce license metadata
26
+ if card.data.license is None:
27
+ if not ("license_name" in card.data and "license_link" in card.data):
28
+ return False, (
29
+ "License not found. Please add a license to your model card using the `license` metadata or a"
30
+ " `license_name`/`license_link` pair."
31
+ )
32
+
33
+ # Enforce card content
34
+ if len(card.text) < 200:
35
+ return False, "Please add a description to your model card, it is too short."
36
+
37
+ return True, ""
38
+
39
+ #
40
+ # def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
41
+ # try:
42
+ # config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
43
+ # if test_tokenizer:
44
+ # try:
45
+ # tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
46
+ # except ValueError as e:
47
+ # return (
48
+ # False,
49
+ # f"uses a tokenizer which is not in a transformers release: {e}",
50
+ # None
51
+ # )
52
+ # except Exception as e:
53
+ # return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
54
+ # return True, None, config
55
+ #
56
+ # except ValueError:
57
+ # return (
58
+ # False,
59
+ # "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
60
+ # None
61
+ # )
62
+ #
63
+ # except Exception as e:
64
+ # return False, "was not found on hub!", None
65
+
66
+
67
+ def get_model_size(model_info: ModelInfo, precision: str):
68
+ size_pattern = size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
69
+ try:
70
+ model_size = round(model_info.safetensors["total"] / 1e9, 3)
71
+ except (AttributeError, TypeError ):
72
+ try:
73
+ size_match = re.search(size_pattern, model_info.modelId.lower())
74
+ model_size = size_match.group(0)
75
+ model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
76
+ except AttributeError:
77
+ return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
78
+
79
+ size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
80
+ model_size = size_factor * model_size
81
+ return model_size
82
+
83
+ def get_model_arch(model_info: ModelInfo):
84
+ return model_info.config.get("architectures", "Unknown")
85
+
86
+ def user_submission_permission(org_or_user, users_to_submission_dates, rate_limit_period, rate_limit_quota):
87
+ if org_or_user not in users_to_submission_dates:
88
+ return True, ""
89
+ submission_dates = sorted(users_to_submission_dates[org_or_user])
90
+
91
+ time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
92
+ submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
93
+
94
+ num_models_submitted_in_period = len(submissions_after_timelimit)
95
+ if org_or_user in HAS_HIGHER_RATE_LIMIT:
96
+ rate_limit_quota = 2 * rate_limit_quota
97
+
98
+ if num_models_submitted_in_period > rate_limit_quota:
99
+ error_msg = f"Organisation or user `{org_or_user}`"
100
+ error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
101
+ error_msg += f"in the last {rate_limit_period} days.\n"
102
+ error_msg += (
103
+ "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
104
+ )
105
+ return False, error_msg
106
+ return True, ""
107
+
108
+
109
+ def already_submitted_models(requested_models_dir: str) -> set[str]:
110
+ depth = 1
111
+ file_names = []
112
+ users_to_submission_dates = defaultdict(list)
113
+
114
+ for root, _, files in os.walk(requested_models_dir):
115
+ current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
116
+ if current_depth == depth:
117
+ for file in files:
118
+ if not file.endswith(".json"):
119
+ continue
120
+ with open(os.path.join(root, file), "r") as f:
121
+ info = json.load(f)
122
+ file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
123
+
124
+ # Select organisation
125
+ if info["model"].count("/") == 0 or "submitted_time" not in info:
126
+ continue
127
+ organisation, _ = info["model"].split("/")
128
+ users_to_submission_dates[organisation].append(info["submitted_time"])
129
+
130
+ return set(file_names), users_to_submission_dates
src/submission/submit.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ from datetime import datetime, timezone
4
+
5
+ from src.display.formatting import styled_error, styled_message, styled_warning
6
+ from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
7
+ from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
8
+ from src.submission.check_validity import (
9
+ already_submitted_models,
10
+ check_model_card,
11
+ get_model_size,
12
+ is_model_on_hub,
13
+ user_submission_permission,
14
+ )
15
+
16
+ REQUESTED_MODELS = None
17
+ USERS_TO_SUBMISSION_DATES = None
18
+
19
+ def add_new_eval(
20
+ model: str,
21
+ base_model: str,
22
+ revision: str,
23
+ precision: str,
24
+ private: bool,
25
+ weight_type: str,
26
+ model_type: str,
27
+ ):
28
+ global REQUESTED_MODELS
29
+ global USERS_TO_SUBMISSION_DATES
30
+ if not REQUESTED_MODELS:
31
+ REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
32
+
33
+ user_name = ""
34
+ model_path = model
35
+ if "/" in model:
36
+ user_name = model.split("/")[0]
37
+ model_path = model.split("/")[1]
38
+
39
+ precision = precision.split(" ")[0]
40
+ current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
41
+
42
+ if model_type is None or model_type == "":
43
+ return styled_error("Please select a model type.")
44
+
45
+ # Is the user rate limited?
46
+ if user_name != "":
47
+ user_can_submit, error_msg = user_submission_permission(
48
+ user_name, USERS_TO_SUBMISSION_DATES, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
49
+ )
50
+ if not user_can_submit:
51
+ return styled_error(error_msg)
52
+
53
+ # Did the model authors forbid its submission to the leaderboard?
54
+ if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS:
55
+ return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
56
+
57
+ # Does the model actually exist?
58
+ if revision == "":
59
+ revision = "main"
60
+
61
+ # Is the model on the hub?
62
+ if weight_type in ["Delta", "Adapter"]:
63
+ base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=H4_TOKEN, test_tokenizer=True)
64
+ if not base_model_on_hub:
65
+ return styled_error(f'Base model "{base_model}" {error}')
66
+
67
+ if not weight_type == "Adapter":
68
+ model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, test_tokenizer=True)
69
+ if not model_on_hub:
70
+ return styled_error(f'Model "{model}" {error}')
71
+
72
+ # Is the model info correctly filled?
73
+ try:
74
+ model_info = API.model_info(repo_id=model, revision=revision)
75
+ except Exception:
76
+ return styled_error("Could not get your model information. Please fill it up properly.")
77
+
78
+ model_size = get_model_size(model_info=model_info, precision=precision)
79
+
80
+ # Were the model card and license filled?
81
+ try:
82
+ license = model_info.cardData["license"]
83
+ except Exception:
84
+ return styled_error("Please select a license for your model")
85
+
86
+ modelcard_OK, error_msg = check_model_card(model)
87
+ if not modelcard_OK:
88
+ return styled_error(error_msg)
89
+
90
+ # Seems good, creating the eval
91
+ print("Adding new eval")
92
+
93
+ eval_entry = {
94
+ "model": model,
95
+ "base_model": base_model,
96
+ "revision": revision,
97
+ "private": private,
98
+ "precision": precision,
99
+ "weight_type": weight_type,
100
+ "status": "PENDING",
101
+ "submitted_time": current_time,
102
+ "model_type": model_type,
103
+ "likes": model_info.likes,
104
+ "params": model_size,
105
+ "license": license,
106
+ }
107
+
108
+ # Check for duplicate submission
109
+ if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
110
+ return styled_warning("This model has been already submitted.")
111
+
112
+ print("Creating eval file")
113
+ OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
114
+ os.makedirs(OUT_DIR, exist_ok=True)
115
+ out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
116
+
117
+ with open(out_path, "w") as f:
118
+ f.write(json.dumps(eval_entry))
119
+
120
+ print("Uploading eval file")
121
+ API.upload_file(
122
+ path_or_fileobj=out_path,
123
+ path_in_repo=out_path.split("eval-queue/")[1],
124
+ repo_id=QUEUE_REPO,
125
+ repo_type="dataset",
126
+ commit_message=f"Add {model} to eval queue",
127
+ )
128
+
129
+ # Remove the local file
130
+ os.remove(out_path)
131
+
132
+ return styled_message(
133
+ "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
134
+ )
src/tools/collections.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ import pandas as pd
4
+ from huggingface_hub import add_collection_item, delete_collection_item, get_collection, update_collection_item
5
+ from huggingface_hub.utils._errors import HfHubHTTPError
6
+ from pandas import DataFrame
7
+
8
+ from src.display.utils import AutoEvalColumn, ModelType
9
+ from src.envs import H4_TOKEN, PATH_TO_COLLECTION
10
+
11
+ # Specific intervals for the collections
12
+ intervals = {
13
+ "1B": pd.Interval(0, 1.5, closed="right"),
14
+ "3B": pd.Interval(2.5, 3.5, closed="neither"),
15
+ "7B": pd.Interval(6, 8, closed="neither"),
16
+ "13B": pd.Interval(10, 14, closed="neither"),
17
+ "30B": pd.Interval(25, 35, closed="neither"),
18
+ "65B": pd.Interval(60, 70, closed="neither"),
19
+ }
20
+
21
+
22
+ def update_collections(df: DataFrame):
23
+ """This function updates the Open LLM Leaderboard model collection with the latest best models for
24
+ each size category and type.
25
+ """
26
+ collection = get_collection(collection_slug=PATH_TO_COLLECTION, token=H4_TOKEN)
27
+ params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
28
+
29
+ cur_best_models = []
30
+
31
+ ix = 0
32
+ for type in ModelType:
33
+ if type.value.name == "":
34
+ continue
35
+ for size in intervals:
36
+ # We filter the df to gather the relevant models
37
+ type_emoji = [t[0] for t in type.value.symbol]
38
+ filtered_df = df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
39
+
40
+ numeric_interval = pd.IntervalIndex([intervals[size]])
41
+ mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
42
+ filtered_df = filtered_df.loc[mask]
43
+
44
+ best_models = list(
45
+ filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.dummy.name]
46
+ )
47
+ print(type.value.symbol, size, best_models[:10])
48
+
49
+ # We add them one by one to the leaderboard
50
+ for model in best_models:
51
+ ix += 1
52
+ cur_len_collection = len(collection.items)
53
+ try:
54
+ collection = add_collection_item(
55
+ PATH_TO_COLLECTION,
56
+ item_id=model,
57
+ item_type="model",
58
+ exists_ok=True,
59
+ note=f"Best {type.to_str(' ')} model of around {size} on the leaderboard today!",
60
+ token=H4_TOKEN,
61
+ )
62
+ if (
63
+ len(collection.items) > cur_len_collection
64
+ ): # we added an item - we make sure its position is correct
65
+ item_object_id = collection.items[-1].item_object_id
66
+ update_collection_item(
67
+ collection_slug=PATH_TO_COLLECTION, item_object_id=item_object_id, position=ix
68
+ )
69
+ cur_len_collection = len(collection.items)
70
+ cur_best_models.append(model)
71
+ break
72
+ except HfHubHTTPError:
73
+ continue
74
+
75
+ collection = get_collection(PATH_TO_COLLECTION, token=H4_TOKEN)
76
+ for item in collection.items:
77
+ if item.item_id not in cur_best_models:
78
+ try:
79
+ delete_collection_item(
80
+ collection_slug=PATH_TO_COLLECTION, item_object_id=item.item_object_id, token=H4_TOKEN
81
+ )
82
+ except HfHubHTTPError:
83
+ continue
src/tools/model_backlinks.py ADDED
@@ -0,0 +1,1309 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ models = [
2
+ "uni-tianyan/Uni-TianYan",
3
+ "fangloveskari/ORCA_LLaMA_70B_QLoRA",
4
+ "garage-bAInd/Platypus2-70B-instruct",
5
+ "upstage/Llama-2-70b-instruct-v2",
6
+ "fangloveskari/Platypus_QLoRA_LLaMA_70b",
7
+ "yeontaek/llama-2-70B-ensemble-v5",
8
+ "TheBloke/Genz-70b-GPTQ",
9
+ "TheBloke/Platypus2-70B-Instruct-GPTQ",
10
+ "psmathur/model_007",
11
+ "yeontaek/llama-2-70B-ensemble-v4",
12
+ "psmathur/orca_mini_v3_70b",
13
+ "ehartford/Samantha-1.11-70b",
14
+ "MayaPH/GodziLLa2-70B",
15
+ "psmathur/model_007_v2",
16
+ "chargoddard/MelangeA-70b",
17
+ "ehartford/Samantha-1.1-70b",
18
+ "psmathur/model_009",
19
+ "upstage/Llama-2-70b-instruct",
20
+ "yeontaek/llama-2-70B-ensemble-v7",
21
+ "yeontaek/llama-2-70B-ensemble-v6",
22
+ "chargoddard/MelangeB-70b",
23
+ "yeontaek/llama-2-70B-ensemble-v3",
24
+ "chargoddard/MelangeC-70b",
25
+ "garage-bAInd/Camel-Platypus2-70B",
26
+ "yeontaek/llama-2-70B-ensemble-v2",
27
+ "garage-bAInd/Camel-Platypus2-70B",
28
+ "migtissera/Synthia-70B-v1.2",
29
+ "v2ray/LLaMA-2-Wizard-70B-QLoRA",
30
+ "quantumaikr/llama-2-70b-fb16-orca-chat-10k",
31
+ "v2ray/LLaMA-2-Wizard-70B-QLoRA",
32
+ "stabilityai/StableBeluga2",
33
+ "quantumaikr/llama-2-70b-fb16-guanaco-1k",
34
+ "garage-bAInd/Camel-Platypus2-70B",
35
+ "migtissera/Synthia-70B-v1.1",
36
+ "migtissera/Synthia-70B",
37
+ "psmathur/model_101",
38
+ "augtoma/qCammel70",
39
+ "augtoma/qCammel-70",
40
+ "augtoma/qCammel-70v1",
41
+ "augtoma/qCammel-70x",
42
+ "augtoma/qCammel-70-x",
43
+ "jondurbin/airoboros-l2-70b-gpt4-1.4.1",
44
+ "dfurman/llama-2-70b-dolphin-peft",
45
+ "jondurbin/airoboros-l2-70b-2.1",
46
+ "TheBloke/llama-2-70b-Guanaco-QLoRA-fp16",
47
+ "quantumaikr/QuantumLM-llama2-70B-Korean-LoRA",
48
+ "quantumaikr/quantumairk-llama-2-70B-instruct",
49
+ "psmathur/model_420",
50
+ "psmathur/model_51",
51
+ "garage-bAInd/Camel-Platypus2-70B",
52
+ "TheBloke/Airoboros-L2-70B-2.1-GPTQ",
53
+ "OpenAssistant/llama2-70b-oasst-sft-v10",
54
+ "garage-bAInd/Platypus2-70B",
55
+ "liuxiang886/llama2-70B-qlora-gpt4",
56
+ "upstage/llama-65b-instruct",
57
+ "quantumaikr/llama-2-70b-fb16-korean",
58
+ "NousResearch/Nous-Hermes-Llama2-70b",
59
+ "v2ray/LLaMA-2-Jannie-70B-QLoRA",
60
+ "jondurbin/airoboros-l2-70b-gpt4-m2.0",
61
+ "jondurbin/airoboros-l2-70b-gpt4-m2.0",
62
+ "OpenAssistant/llama2-70b-oasst-sft-v10",
63
+ "yeontaek/llama-2-70B-ensemble-v8",
64
+ "jondurbin/airoboros-l2-70b-gpt4-2.0",
65
+ "jarradh/llama2_70b_chat_uncensored",
66
+ "WizardLM/WizardMath-70B-V1.0",
67
+ "jordiclive/Llama-2-70b-oasst-1-200",
68
+ "WizardLM/WizardMath-70B-V1.0",
69
+ "jondurbin/airoboros-l2-70b-gpt4-2.0",
70
+ "OpenLemur/lemur-70b-chat-v1",
71
+ "tiiuae/falcon-180B",
72
+ "tiiuae/falcon-180B",
73
+ "stabilityai/StableBeluga1-Delta",
74
+ "psmathur/model_42_70b",
75
+ "psmathur/test_42_70b",
76
+ "TheBloke/fiction.live-Kimiko-V2-70B-fp16",
77
+ "tiiuae/falcon-180B",
78
+ "WizardLM/WizardMath-70B-V1.0",
79
+ "tiiuae/falcon-180B-chat",
80
+ "jondurbin/airoboros-l2-70b-gpt4-2.0",
81
+ "ehartford/samantha-1.1-llama-33b",
82
+ "ajibawa-2023/scarlett-33b",
83
+ "ddobokki/Llama-2-70b-orca-200k",
84
+ "TheBloke/gpt4-alpaca-lora_mlp-65B-HF",
85
+ "tiiuae/falcon-180B-chat",
86
+ "tiiuae/falcon-180B-chat",
87
+ "tiiuae/falcon-180B",
88
+ "TheBloke/Lemur-70B-Chat-v1-GPTQ",
89
+ "NousResearch/Nous-Puffin-70B",
90
+ "WizardLM/WizardLM-70B-V1.0",
91
+ "WizardLM/WizardMath-70B-V1.0",
92
+ "meta-llama/Llama-2-70b-hf",
93
+ "TheBloke/Llama-2-70B-fp16",
94
+ "Weyaxi/llama-2-alpacagpt4-1000step",
95
+ "WizardLM/WizardLM-70B-V1.0",
96
+ "simsim314/WizardLM-70B-V1.0-HF",
97
+ "simsim314/WizardLM-70B-V1.0-HF",
98
+ "WizardLM/WizardLM-70B-V1.0",
99
+ "openbmb/UltraLM-65b",
100
+ "psmathur/model_420_preview",
101
+ "WizardLM/WizardLM-70B-V1.0",
102
+ "simsim314/WizardLM-70B-V1.0-HF",
103
+ "OpenBuddy/openbuddy-llama2-70b-v10.1-bf16",
104
+ "upstage/llama-30b-instruct-2048",
105
+ "jondurbin/airoboros-65b-gpt4-1.2",
106
+ "TheBloke/guanaco-65B-HF",
107
+ "jondurbin/airoboros-65b-gpt4-1.3",
108
+ "meta-llama/Llama-2-70b-chat-hf",
109
+ "ValiantLabs/ShiningValiant",
110
+ "Faradaylab/Aria-70B",
111
+ "lilloukas/GPlatty-30B",
112
+ "TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16",
113
+ "jondurbin/airoboros-65b-gpt4-1.4-peft",
114
+ "jondurbin/airoboros-65b-gpt4-1.4",
115
+ "jondurbin/airoboros-65b-gpt4-2.0",
116
+ "TheBloke/WizardLM-70B-V1.0-GPTQ",
117
+ "TheBloke/WizardLM-70B-V1.0-GPTQ",
118
+ "ariellee/SuperPlatty-30B",
119
+ "jondurbin/airoboros-65b-gpt4-1.4",
120
+ "jondurbin/airoboros-65b-gpt4-2.0",
121
+ "yeontaek/llama-2-70b-IA3-guanaco",
122
+ "CalderaAI/30B-Lazarus",
123
+ "Aspik101/trurl-2-13b-pl-instruct_unload",
124
+ "ehartford/WizardLM-33B-V1.0-Uncensored",
125
+ "ehartford/WizardLM-33B-V1.0-Uncensored",
126
+ "OpenBuddy/openbuddy-llama-65b-v8-bf16",
127
+ "Aspik101/llama-30b-instruct-2048-PL-lora",
128
+ "h2oai/h2ogpt-research-oasst1-llama-65b",
129
+ "Aspik101/llama-30b-instruct-2048-PL-lora",
130
+ "CalderaAI/30B-Epsilon",
131
+ "Aspik101/llama-30b-2048-instruct-PL-lora_unload",
132
+ "jondurbin/airoboros-65b-gpt4-m2.0",
133
+ "jondurbin/airoboros-65b-gpt4-m2.0",
134
+ "Aeala/Alpaca-elina-65b",
135
+ "TheBloke/robin-65b-v2-fp16",
136
+ "TheBloke/gpt4-alpaca-lora-30b-HF",
137
+ "TheBloke/Llama-2-70B-chat-GPTQ",
138
+ "upstage/llama-30b-instruct",
139
+ "OpenLemur/lemur-70b-v1",
140
+ "lmsys/vicuna-33b-v1.3",
141
+ "ausboss/llama-30b-supercot",
142
+ "ai-business/Luban-13B",
143
+ "Henk717/airochronos-33B",
144
+ "lmsys/vicuna-33b-v1.3",
145
+ "Henk717/airochronos-33B",
146
+ "bavest/fin-llama-33b-merged",
147
+ "jondurbin/airoboros-33b-gpt4-1.4",
148
+ "YeungNLP/firefly-llama-30b",
149
+ "Aspik101/30B-Lazarus-instruct-PL-lora_unload",
150
+ "uukuguy/speechless-llama2-luban-orca-platypus-13b",
151
+ "xxyyy123/test_merge_p_ov1_w0.66_w0.5_n1",
152
+ "jondurbin/airoboros-33b-gpt4-1.2",
153
+ "TheBloke/alpaca-lora-65B-HF",
154
+ "bofenghuang/vigogne-33b-instruct",
155
+ "yeontaek/llama-2-13B-ensemble-v5",
156
+ "garage-bAInd/Platypus-30B",
157
+ "Open-Orca/OpenOrca-Platypus2-13B",
158
+ "kajdun/viwaai-30b_v4",
159
+ "lilloukas/Platypus-30B",
160
+ "Open-Orca/OpenOrca-Platypus2-13B",
161
+ "Henk717/chronoboros-33B",
162
+ "jondurbin/airoboros-33b-2.1",
163
+ "HiTZ/alpaca-lora-65b-en-pt-es-ca",
164
+ "quantumaikr/QuantumLM-70B-hf",
165
+ "uukuguy/speechless-llama2-13b",
166
+ "uukuguy/speechless-llama2-hermes-orca-platypus-13b",
167
+ "openaccess-ai-collective/manticore-30b-chat-pyg-alpha",
168
+ "LLMs/WizardLM-30B-V1.0",
169
+ "TheBloke/WizardLM-30B-fp16",
170
+ "openaccess-ai-collective/hippogriff-30b-chat",
171
+ "concedo/Vicuzard-30B-Uncensored",
172
+ "TFLai/OpenOrca-Platypus2-13B-QLoRA-0.80-epoch",
173
+ "huggingface/llama-65b",
174
+ "huggyllama/llama-65b",
175
+ "gaodrew/gaodrew-llama-30b-instruct-2048-Open-Platypus-100steps",
176
+ "uukuguy/speechless-llama2-hermes-orca-platypus-wizardlm-13b",
177
+ "Sao10K/Mythical-Destroyer-V2-L2-13B",
178
+ "camel-ai/CAMEL-33B-Combined-Data",
179
+ "dsvv-cair/alpaca-cleaned-llama-30b-bf16",
180
+ "MetaIX/GPT4-X-Alpasta-30b",
181
+ "garage-bAInd/Stable-Platypus2-13B",
182
+ "TFLai/Luban-Platypus2-13B-QLora-0.80-epoch",
183
+ "TheBloke/OpenOrca-Platypus2-13B-GPTQ",
184
+ "IkariDev/Athena-tmp",
185
+ "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16",
186
+ "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16",
187
+ "Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
188
+ "psmathur/model_007_13b_v2",
189
+ "Aspik101/Vicuzard-30B-Uncensored-instruct-PL-lora_unload",
190
+ "jondurbin/airoboros-33b-gpt4-m2.0",
191
+ "Sao10K/Mythical-Destroyer-L2-13B",
192
+ "TheBloke/Wizard-Vicuna-30B-Uncensored-fp16",
193
+ "ehartford/Wizard-Vicuna-30B-Uncensored",
194
+ "TFLai/Nova-13B",
195
+ "TheBloke/robin-33B-v2-fp16",
196
+ "totally-not-an-llm/PuddleJumper-13b",
197
+ "Aeala/VicUnlocked-alpaca-30b",
198
+ "Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf",
199
+ "jondurbin/airoboros-33b-gpt4",
200
+ "jondurbin/airoboros-33b-gpt4-m2.0",
201
+ "tiiuae/falcon-40b-instruct",
202
+ "psmathur/orca_mini_v3_13b",
203
+ "Aeala/GPT4-x-AlpacaDente-30b",
204
+ "MayaPH/GodziLLa-30B",
205
+ "jondurbin/airoboros-33b-gpt4-m2.0",
206
+ "TFLai/SpeechlessV1-Nova-13B",
207
+ "yeontaek/llama-2-13B-ensemble-v4",
208
+ "ajibawa-2023/carl-33b",
209
+ "jondurbin/airoboros-33b-gpt4-2.0",
210
+ "TFLai/Stable-Platypus2-13B-QLoRA-0.80-epoch",
211
+ "jondurbin/airoboros-33b-gpt4-1.3",
212
+ "TehVenom/oasst-sft-6-llama-33b-xor-MERGED-16bit",
213
+ "TFLai/OrcaMini-Platypus2-13B-QLoRA-0.80-epoch",
214
+ "jondurbin/airoboros-33b-gpt4-2.0",
215
+ "chargoddard/Chronorctypus-Limarobormes-13b",
216
+ "jondurbin/airoboros-33b-gpt4-1.3",
217
+ "Open-Orca/OpenOrca-Platypus2-13B",
218
+ "FelixChao/vicuna-33b-coder",
219
+ "FelixChao/vicuna-33b-coder",
220
+ "Gryphe/MythoMix-L2-13b",
221
+ "Aeala/Enterredaas-33b",
222
+ "yeontaek/llama-2-13B-ensemble-v1",
223
+ "TFLai/OpenOrcaPlatypus2-Platypus2-13B-QLora-0.80-epoch",
224
+ "TFLai/Ensemble5-Platypus2-13B-QLora-0.80-epoch",
225
+ "yeontaek/llama-2-13B-ensemble-v3",
226
+ "TFLai/MythoMix-Platypus2-13B-QLoRA-0.80-epoch",
227
+ "yihan6324/llama2-13b-instructmining-40k-sharegpt",
228
+ "timdettmers/guanaco-33b-merged",
229
+ "TFLai/EnsembleV5-Nova-13B",
230
+ "circulus/Llama-2-13b-orca-v1",
231
+ "Undi95/ReMM-SLERP-L2-13B",
232
+ "Gryphe/MythoMax-L2-13b",
233
+ "stabilityai/StableBeluga-13B",
234
+ "circulus/Llama-2-13b-orca-v1",
235
+ "ehartford/WizardLM-30B-Uncensored",
236
+ "The-Face-Of-Goonery/huginnv1.2",
237
+ "TheBloke/OpenOrcaxOpenChat-Preview2-13B-GPTQ",
238
+ "Sao10K/Stheno-L2-13B",
239
+ "bofenghuang/vigogne-2-13b-instruct",
240
+ "The-Face-Of-Goonery/Huginn-13b-FP16",
241
+ "grimpep/L2-MythoMax22b-instruct-Falseblock",
242
+ "TFLai/Nous-Hermes-Platypus2-13B-QLoRA-0.80-epoch",
243
+ "yeontaek/Platypus2xOpenOrca-13B-IA3-v4",
244
+ "yeontaek/Platypus2xOpenOrca-13B-IA3",
245
+ "yeontaek/Platypus2xOpenOrca-13B-IA3-ensemble",
246
+ "Open-Orca/LlongOrca-13B-16k",
247
+ "Sao10K/Stheno-Inverted-L2-13B",
248
+ "garage-bAInd/Camel-Platypus2-13B",
249
+ "digitous/Alpacino30b",
250
+ "NousResearch/Nous-Hermes-Llama2-13b",
251
+ "yeontaek/Platypus2xOpenOrca-13B-IA3-v3",
252
+ "TFLai/MythicalDestroyerV2-Platypus2-13B-QLora-0.80-epoch",
253
+ "TheBloke/VicUnlocked-30B-LoRA-HF",
254
+ "Undi95/Nous-Hermes-13B-Code",
255
+ "The-Face-Of-Goonery/Chronos-Beluga-v2-13bfp16",
256
+ "NousResearch/Nous-Hermes-Llama2-13b",
257
+ "Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b",
258
+ "TheBloke/Wizard-Vicuna-30B-Uncensored-GPTQ",
259
+ "Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
260
+ "Austism/chronos-hermes-13b-v2",
261
+ "yeontaek/Platypus2xOpenOrca-13B-IA3-v2.1",
262
+ "yeontaek/Platypus2xOpenOrca-13B-IA3-v2",
263
+ "Gryphe/MythoLogic-L2-13b",
264
+ "augtoma/qCammel-13",
265
+ "YeungNLP/firefly-llama2-13b-v1.2",
266
+ "Aspik101/StableBeluga-13B-instruct-PL-lora_unload",
267
+ "andreaskoepf/llama2-13b-megacode2_min100",
268
+ "rombodawg/LosslessMegaCoder-llama2-13b-mini",
269
+ "yulan-team/YuLan-Chat-2-13b-fp16",
270
+ "elinas/chronos-33b",
271
+ "YeungNLP/firefly-llama2-13b",
272
+ "Sao10K/Medusa-13b",
273
+ "OptimalScale/robin-65b-v2-delta",
274
+ "minlik/chinese-alpaca-33b-merged",
275
+ "OpenAssistant/llama2-13b-megacode2-oasst",
276
+ "TheBloke/OpenAssistant-SFT-7-Llama-30B-HF",
277
+ "Undi95/UndiMix-v1-13b",
278
+ "ehartford/Samantha-1.11-13b",
279
+ "beaugogh/Llama2-13b-sharegpt4",
280
+ "Aeala/GPT4-x-AlpacaDente2-30b",
281
+ "luffycodes/nash-vicuna-13b-v1dot5-ep2-w-rag-w-simple",
282
+ "WizardLM/WizardLM-13B-V1.1",
283
+ "uukuguy/speechless-orca-platypus-coig-lite-2k-0.6e-13b",
284
+ "huggyllama/llama-30b",
285
+ "Undi95/ReMM-L2-13B-PIPPA",
286
+ "Undi95/ReMM-L2-13B",
287
+ "gaodrew/gaodrew-gorgonzola-13b",
288
+ "lmsys/vicuna-13b-v1.5",
289
+ "yeontaek/Platypus2xOpenOrca-13B-LoRa",
290
+ "Yhyu13/llama-30B-hf-openassitant",
291
+ "huggingface/llama-30b",
292
+ "lmsys/vicuna-13b-v1.5",
293
+ "TFLai/Athena-Platypus2-13B-QLora-0.80-epoch",
294
+ "TheBloke/dromedary-65b-lora-HF",
295
+ "yeontaek/llama-2-13b-Beluga-QLoRA",
296
+ "The-Face-Of-Goonery/Huginn-13b-V4",
297
+ "The-Face-Of-Goonery/Huginn-13b-v4.5",
298
+ "The-Face-Of-Goonery/Huginn-v3-13b",
299
+ "tiiuae/falcon-40b",
300
+ "WhoTookMyAmogusNickname/NewHope_HF_not_official",
301
+ "gaodrew/OpenOrca-Platypus2-13B-thera-1250",
302
+ "SLAM-group/NewHope",
303
+ "garage-bAInd/Platypus2-13B",
304
+ "migtissera/Synthia-13B",
305
+ "elinas/chronos-13b-v2",
306
+ "mosaicml/mpt-30b-chat",
307
+ "CHIH-HUNG/llama-2-13b-OpenOrca_5w",
308
+ "uukuguy/speechless-hermes-coig-lite-13b",
309
+ "TheBloke/tulu-30B-fp16",
310
+ "uukuguy/speechless-hermes-coig-lite-13b",
311
+ "xDAN-AI/xDAN_13b_l2_lora",
312
+ "lmsys/vicuna-13b-v1.5-16k",
313
+ "openchat/openchat_v3.1",
314
+ "CHIH-HUNG/llama-2-13b-dolphin_5w",
315
+ "Aspik101/vicuna-13b-v1.5-PL-lora_unload",
316
+ "Undi95/MLewd-L2-13B",
317
+ "ehartford/minotaur-llama2-13b-qlora",
318
+ "kajdun/iubaris-13b-v3",
319
+ "TFLai/Limarp-Platypus2-13B-QLoRA-0.80-epoch",
320
+ "openchat/openchat_v3.1",
321
+ "uukuguy/speechless-orca-platypus-coig-lite-4k-0.6e-13b",
322
+ "ziqingyang/chinese-alpaca-2-13b",
323
+ "TFLai/Airboros2.1-Platypus2-13B-QLora-0.80-epoch",
324
+ "yeontaek/llama-2-13b-Guanaco-QLoRA",
325
+ "lmsys/vicuna-13b-v1.5-16k",
326
+ "ehartford/based-30b",
327
+ "kingbri/airolima-chronos-grad-l2-13B",
328
+ "openchat/openchat_v3.2",
329
+ "uukuguy/speechless-orca-platypus-coig-lite-4k-0.5e-13b",
330
+ "yeontaek/Platypus2-13B-LoRa",
331
+ "kingbri/chronolima-airo-grad-l2-13B",
332
+ "openchat/openchat_v3.2",
333
+ "TFLai/PuddleJumper-Platypus2-13B-QLoRA-0.80-epoch",
334
+ "shareAI/llama2-13b-Chinese-chat",
335
+ "ehartford/WizardLM-1.0-Uncensored-Llama2-13b",
336
+ "Aspik101/Redmond-Puffin-13B-instruct-PL-lora_unload",
337
+ "yeontaek/llama-2-13B-ensemble-v6",
338
+ "WizardLM/WizardLM-13B-V1.2",
339
+ "TheBloke/WizardLM-13B-V1.1-GPTQ",
340
+ "bhenrym14/airophin-13b-pntk-16k-fp16",
341
+ "ehartford/WizardLM-1.0-Uncensored-Llama2-13b",
342
+ "Mikael110/llama-2-13b-guanaco-fp16",
343
+ "yeontaek/airoboros-2.1-llama-2-13B-QLoRa",
344
+ "CalderaAI/13B-Legerdemain-L2",
345
+ "grimpep/llama2-22b-wizard_vicuna",
346
+ "grimpep/llama2-22B-GPLATTY",
347
+ "bhenrym14/airophin-13b-pntk-16k-fp16",
348
+ "yeontaek/llama-2-13b-QLoRA",
349
+ "OpenAssistant/llama2-13b-orca-8k-3319",
350
+ "TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16",
351
+ "duliadotio/dulia-13b-8k-alpha",
352
+ "Undi95/LewdEngine",
353
+ "OpenBuddy/openbuddy-llama2-13b-v8.1-fp16",
354
+ "CHIH-HUNG/llama-2-13b-open_orca_20w",
355
+ "bhenrym14/airoboros-33b-gpt4-1.4.1-lxctx-PI-16384-fp16",
356
+ "FlagAlpha/Llama2-Chinese-13b-Chat",
357
+ "LLMs/WizardLM-13B-V1.0",
358
+ "chansung/gpt4-alpaca-lora-13b-decapoda-1024",
359
+ "TheBloke/wizardLM-13B-1.0-fp16",
360
+ "digitous/13B-Chimera",
361
+ "yeontaek/Platypus2xOpenOrcaxGuanaco-13B-LoRa",
362
+ "jondurbin/airoboros-l2-13b-2.1",
363
+ "Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b",
364
+ "TheBloke/UltraLM-13B-fp16",
365
+ "openaccess-ai-collective/minotaur-13b-fixed",
366
+ "NousResearch/Redmond-Puffin-13B",
367
+ "KoboldAI/LLaMA2-13B-Holomax",
368
+ "Lajonbot/WizardLM-13B-V1.2-PL-lora_unload",
369
+ "yeontaek/Platypus2-13B-LoRa-v2",
370
+ "TheBloke/airoboros-13B-HF",
371
+ "jondurbin/airoboros-13b",
372
+ "jjaaaww/posi_13b",
373
+ "CoolWP/llama-2-13b-guanaco-fp16",
374
+ "yeontaek/Platypus2-13B-QLoRa",
375
+ "h2oai/h2ogpt-research-oig-oasst1-512-30b",
376
+ "dfurman/llama-2-13b-guanaco-peft",
377
+ "NousResearch/Redmond-Puffin-13B",
378
+ "pe-nlp/llama-2-13b-platypus-vicuna-wizard",
379
+ "CHIH-HUNG/llama-2-13b-dolphin_20w",
380
+ "NousResearch/Nous-Hermes-13b",
381
+ "NobodyExistsOnTheInternet/GiftedConvo13bLoraNoEconsE4",
382
+ "ehartford/Wizard-Vicuna-13B-Uncensored",
383
+ "TheBloke/Wizard-Vicuna-13B-Uncensored-HF",
384
+ "openchat/openchat_v3.2_super",
385
+ "bhenrym14/airophin-v2-13b-PI-8k-fp16",
386
+ "openaccess-ai-collective/manticore-13b",
387
+ "The-Face-Of-Goonery/Huginn-22b-Prototype",
388
+ "jphme/Llama-2-13b-chat-german",
389
+ "grimpep/llama2-28B-Airo03",
390
+ "TheBloke/Kimiko-v2-13B-fp16",
391
+ "FPHam/Free_Sydney_13b_HF",
392
+ "lmsys/vicuna-13b-v1.3",
393
+ "FelixChao/llama2-13b-math1.1",
394
+ "CalderaAI/13B-BlueMethod",
395
+ "meta-llama/Llama-2-13b-chat-hf",
396
+ "deepse/CodeUp-Llama-2-13b-chat-hf",
397
+ "WizardLM/WizardMath-13B-V1.0",
398
+ "WizardLM/WizardMath-13B-V1.0",
399
+ "HyperbeeAI/Tulpar-7b-v0",
400
+ "xxyyy123/test_qkvo_adptor",
401
+ "xxyyy123/mc_data_30k_from_platpus_orca_7b_10k_v1_lora_qkvo_rank14_v2",
402
+ "openchat/openchat_v2_w",
403
+ "FelixChao/llama2-13b-math1.1",
404
+ "psmathur/orca_mini_v3_7b",
405
+ "TehVenom/Metharme-13b-Merged",
406
+ "xxyyy123/10k_v1_lora_qkvo_rank14_v3",
407
+ "OpenAssistant/llama2-13b-orca-v2-8k-3166",
408
+ "openaccess-ai-collective/wizard-mega-13b",
409
+ "jondurbin/airoboros-13b-gpt4-1.4",
410
+ "jondurbin/airoboros-13b-gpt4-1.4-fp16",
411
+ "Monero/Manticore-13b-Chat-Pyg-Guanaco",
412
+ "FelixChao/llama2-13b-math1.2",
413
+ "chargoddard/platypus-2-22b-relora",
414
+ "FelixChao/llama2-13b-math1.2",
415
+ "Gryphe/MythoBoros-13b",
416
+ "CalderaAI/13B-Ouroboros",
417
+ "OpenAssistant/llama2-13b-orca-v2-8k-3166",
418
+ "heegyu/LIMA2-13b-hf",
419
+ "digitous/13B-HyperMantis",
420
+ "Gryphe/MythoLogic-13b",
421
+ "TheBloke/Airoboros-L2-13B-2.1-GPTQ",
422
+ "chargoddard/platypus2-22b-relora",
423
+ "openchat/openchat_v2",
424
+ "yeontaek/Platypus2-13B-IA3",
425
+ "stabilityai/StableBeluga-7B",
426
+ "circulus/Llama-2-7b-orca-v1",
427
+ "budecosystem/genz-13b-v2",
428
+ "TheBloke/gpt4-x-vicuna-13B-HF",
429
+ "NobodyExistsOnTheInternet/GiftedConvo13bLoraNoEcons",
430
+ "zarakiquemparte/zarafusionex-1.1-l2-7b",
431
+ "Lajonbot/tableBeluga-7B-instruct-pl-lora_unload",
432
+ "jondurbin/airoboros-13b-gpt4",
433
+ "gaodrew/gaodrew-gorgonzola-13b",
434
+ "jondurbin/airoboros-13b-gpt4-1.1",
435
+ "TheBloke/gpt4-alpaca-lora-13B-HF",
436
+ "zarakiquemparte/zarablendex-vq-l2-7b",
437
+ "openaccess-ai-collective/manticore-13b-chat-pyg",
438
+ "Lajonbot/Llama-2-13b-hf-instruct-pl-lora_unload",
439
+ "NobodyExistsOnTheInternet/PuffedLIMA13bQLORA",
440
+ "xxyyy123/10k_v1_lora_qkvo_rank28_v2",
441
+ "jondurbin/airoboros-l2-13b-gpt4-1.4.1",
442
+ "dhmeltzer/Llama-2-13b-hf-eli5-wiki-1024_r_64_alpha_16",
443
+ "NobodyExistsOnTheInternet/PuffedConvo13bLoraE4",
444
+ "yihan6324/llama2-7b-instructmining-40k-sharegpt",
445
+ "CHIH-HUNG/llama-2-13b-Open_Platypus_and_ccp_2.6w",
446
+ "Aeala/GPT4-x-Alpasta-13b",
447
+ "psmathur/orca_mini_v2_13b",
448
+ "YeungNLP/firefly-llama-13b",
449
+ "psmathur/orca_mini_v2_13b",
450
+ "zarakiquemparte/zarafusionix-l2-7b",
451
+ "yihan6324/llama2-7b-instructmining-60k-sharegpt",
452
+ "yihan6324/llama-2-7b-instructmining-60k-sharegpt",
453
+ "layoric/llama-2-13b-code-alpaca",
454
+ "bofenghuang/vigogne-13b-instruct",
455
+ "Lajonbot/vicuna-13b-v1.3-PL-lora_unload",
456
+ "lvkaokao/llama2-7b-hf-chat-lora-v3",
457
+ "ehartford/dolphin-llama-13b",
458
+ "YeungNLP/firefly-llama-13b-v1.2",
459
+ "TheBloke/Kimiko-13B-fp16",
460
+ "kevinpro/Vicuna-13B-CoT",
461
+ "eachadea/vicuna-13b-1.1",
462
+ "pillowtalks-ai/delta13b",
463
+ "TheBloke/vicuna-13B-1.1-HF",
464
+ "TheBloke/Vicuna-13B-CoT-fp16",
465
+ "lmsys/vicuna-13b-delta-v1.1",
466
+ "lmsys/vicuna-13b-v1.1",
467
+ "xxyyy123/20k_v1_lora_qkvo_rank14_v2",
468
+ "TheBloke/guanaco-13B-HF",
469
+ "TheBloke/vicuna-13b-v1.3.0-GPTQ",
470
+ "edor/Stable-Platypus2-mini-7B",
471
+ "totally-not-an-llm/EverythingLM-13b-V2-16k",
472
+ "zarakiquemparte/zaraxe-l2-7b",
473
+ "beaugogh/Llama2-7b-openorca-mc-v2",
474
+ "TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16",
475
+ "quantumaikr/QuantumLM",
476
+ "jondurbin/airoboros-13b-gpt4-1.2",
477
+ "TheBloke/robin-13B-v2-fp16",
478
+ "TFLai/llama-2-13b-4bit-alpaca-gpt4",
479
+ "yihan6324/llama2-7b-instructmining-orca-40k",
480
+ "dvruette/oasst-llama-13b-2-epochs",
481
+ "Open-Orca/LlongOrca-7B-16k",
482
+ "Aspik101/Nous-Hermes-13b-pl-lora_unload",
483
+ "ehartford/Samantha-1.11-CodeLlama-34b",
484
+ "nkpz/llama2-22b-chat-wizard-uncensored",
485
+ "bofenghuang/vigogne-13b-chat",
486
+ "beaugogh/Llama2-7b-openorca-mc-v1",
487
+ "OptimalScale/robin-13b-v2-delta",
488
+ "pe-nlp/llama-2-13b-vicuna-wizard",
489
+ "chargoddard/llama2-22b",
490
+ "gywy/llama2-13b-chinese-v1",
491
+ "frank098/Wizard-Vicuna-13B-juniper",
492
+ "IGeniusDev/llama13B-quant8-testv1-openorca-customdataset",
493
+ "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w-gate_up_down_proj",
494
+ "eachadea/vicuna-13b",
495
+ "yihan6324/llama2-7b-instructmining-orca-90k",
496
+ "chargoddard/llama2-22b-blocktriangular",
497
+ "luffycodes/mcq-vicuna-13b-v1.5",
498
+ "Yhyu13/chimera-inst-chat-13b-hf",
499
+ "luffycodes/mcq-vicuna-13b-v1.5",
500
+ "chargoddard/ypotryll-22b-epoch2-qlora",
501
+ "totally-not-an-llm/EverythingLM-13b-16k",
502
+ "luffycodes/mcq-hal-vicuna-13b-v1.5",
503
+ "openaccess-ai-collective/minotaur-13b",
504
+ "IGeniusDev/llama13B-quant8-testv1-openorca-customdataset",
505
+ "chargoddard/llama2-22b-blocktriangular",
506
+ "TFLai/Platypus2-13B-QLoRA-0.80-epoch",
507
+ "meta-llama/Llama-2-13b-hf",
508
+ "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w-gate_up_down_proj",
509
+ "luffycodes/mcq-hal-vicuna-13b-v1.5",
510
+ "TheBloke/Llama-2-13B-fp16",
511
+ "TaylorAI/Flash-Llama-13B",
512
+ "shareAI/bimoGPT-llama2-13b",
513
+ "wahaha1987/llama_13b_sharegpt94k_fastchat",
514
+ "openchat/openchat_8192",
515
+ "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w-q_k_v_o_proj",
516
+ "dvruette/llama-13b-pretrained-sft-do2",
517
+ "CHIH-HUNG/llama-2-13b-alpaca-test",
518
+ "OpenBuddy/openbuddy-llama2-13b-v11.1-bf16",
519
+ "CHIH-HUNG/llama-2-13b-FINETUNE2_TEST_2.2w",
520
+ "project-baize/baize-v2-13b",
521
+ "jondurbin/airoboros-l2-13b-gpt4-m2.0",
522
+ "yeontaek/Platypus2xOpenOrca-13B-LoRa-v2",
523
+ "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w",
524
+ "xzuyn/Alpacino-SuperCOT-13B",
525
+ "jondurbin/airoboros-l2-13b-gpt4-2.0",
526
+ "aiplanet/effi-13b",
527
+ "clibrain/Llama-2-13b-ft-instruct-es",
528
+ "CHIH-HUNG/llama-2-13b-huangyt_Fintune_1_17w",
529
+ "bofenghuang/vigogne-2-7b-instruct",
530
+ "CHIH-HUNG/llama-2-13b-huangyt_FINETUNE2_3w-q_k_v_o_proj",
531
+ "bofenghuang/vigogne-2-7b-chat",
532
+ "aiplanet/effi-13b",
533
+ "haonan-li/bactrian-x-llama-13b-merged",
534
+ "beaugogh/Llama2-7b-sharegpt4",
535
+ "HWERI/Llama2-7b-sharegpt4",
536
+ "jondurbin/airoboros-13b-gpt4-1.3",
537
+ "jondurbin/airoboros-c34b-2.1",
538
+ "junelee/wizard-vicuna-13b",
539
+ "TheBloke/wizard-vicuna-13B-HF",
540
+ "Open-Orca/OpenOrca-Preview1-13B",
541
+ "TheBloke/h2ogpt-oasst1-512-30B-HF",
542
+ "TheBloke/Llama-2-13B-GPTQ",
543
+ "camel-ai/CAMEL-13B-Combined-Data",
544
+ "lmsys/vicuna-7b-v1.5",
545
+ "lmsys/vicuna-7b-v1.5-16k",
546
+ "lmsys/vicuna-7b-v1.5",
547
+ "ausboss/llama-13b-supercot",
548
+ "TheBloke/tulu-13B-fp16",
549
+ "NousResearch/Nous-Hermes-llama-2-7b",
550
+ "jlevin/guanaco-13b-llama-2",
551
+ "lmsys/vicuna-7b-v1.5-16k",
552
+ "dvruette/llama-13b-pretrained",
553
+ "nkpz/llama2-22b-daydreamer-v3",
554
+ "dvruette/llama-13b-pretrained-dropout",
555
+ "jondurbin/airoboros-l2-13b-2.1",
556
+ "LLMs/Stable-Vicuna-13B",
557
+ "64bits/LexPodLM-13B",
558
+ "lizhuang144/llama_mirror_13b_v1.0",
559
+ "TheBloke/stable-vicuna-13B-HF",
560
+ "zarakiquemparte/zaraxls-l2-7b",
561
+ "TheBloke/Llama-2-13B-GPTQ",
562
+ "Kiddyz/testlm-3",
563
+ "migtissera/Synthia-7B",
564
+ "zarakiquemparte/zarablend-l2-7b",
565
+ "mosaicml/mpt-30b-instruct",
566
+ "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged",
567
+ "vonjack/Qwen-LLaMAfied-HFTok-7B-Chat",
568
+ "l3utterfly/llama2-7b-layla",
569
+ "Lajonbot/vicuna-7b-v1.5-PL-lora_unload",
570
+ "heegyu/LIMA-13b-hf",
571
+ "frank098/WizardLM_13B_juniper",
572
+ "ashercn97/manatee-7b",
573
+ "chavinlo/gpt4-x-alpaca",
574
+ "PocketDoc/Dans-PersonalityEngine-13b",
575
+ "ehartford/WizardLM-1.0-Uncensored-CodeLlama-34b",
576
+ "digitous/Alpacino13b",
577
+ "edor/Hermes-Platypus2-mini-7B",
578
+ "lvkaokao/llama2-7b-hf-chat-lora-v2",
579
+ "Kiddyz/testlm-1-1",
580
+ "Kiddyz/testlm",
581
+ "Kiddyz/testlm-1",
582
+ "Kiddyz/testlm2",
583
+ "radm/Philosophy-Platypus2-13b",
584
+ "aiplanet/effi-13b",
585
+ "Harshvir/Llama-2-7B-physics",
586
+ "YeungNLP/firefly-ziya-13b",
587
+ "LinkSoul/Chinese-Llama-2-7b",
588
+ "PeanutJar/LLaMa-2-PeanutButter_v10-7B",
589
+ "OpenBuddy/openbuddy-llama2-13b-v11-bf16",
590
+ "StudentLLM/Alpagasus-2-13B-QLoRA-pipeline",
591
+ "meta-llama/Llama-2-13b-hf",
592
+ "WizardLM/WizardCoder-Python-34B-V1.0",
593
+ "dvruette/llama-13b-pretrained-sft-epoch-1",
594
+ "camel-ai/CAMEL-13B-Role-Playing-Data",
595
+ "ziqingyang/chinese-llama-2-13b",
596
+ "rombodawg/LosslessMegaCoder-llama2-7b-mini",
597
+ "TheBloke/koala-13B-HF",
598
+ "lmsys/vicuna-7b-delta-v1.1",
599
+ "eachadea/vicuna-7b-1.1",
600
+ "Ejafa/vicuna_7B_vanilla_1.1",
601
+ "lvkaokao/llama2-7b-hf-chat-lora",
602
+ "OpenBuddy/openbuddy-atom-13b-v9-bf16",
603
+ "Norquinal/llama-2-7b-claude-chat-rp",
604
+ "Danielbrdz/Barcenas-7b",
605
+ "heegyu/WizardVicuna2-13b-hf",
606
+ "meta-llama/Llama-2-7b-chat-hf",
607
+ "PeanutJar/LLaMa-2-PeanutButter_v14-7B",
608
+ "PeanutJar/LLaMa-2-PeanutButter_v4-7B",
609
+ "davzoku/cria-llama2-7b-v1.3",
610
+ "OpenBuddy/openbuddy-atom-13b-v9-bf16",
611
+ "lvkaokao/llama2-7b-hf-instruction-lora",
612
+ "Tap-M/Luna-AI-Llama2-Uncensored",
613
+ "ehartford/Samantha-1.11-7b",
614
+ "WizardLM/WizardCoder-Python-34B-V1.0",
615
+ "TheBloke/Manticore-13B-Chat-Pyg-Guanaco-SuperHOT-8K-GPTQ",
616
+ "Mikael110/llama-2-7b-guanaco-fp16",
617
+ "garage-bAInd/Platypus2-7B",
618
+ "PeanutJar/LLaMa-2-PeanutButter_v18_B-7B",
619
+ "mosaicml/mpt-30b",
620
+ "garage-bAInd/Platypus2-7B",
621
+ "huggingface/llama-13b",
622
+ "dvruette/oasst-llama-13b-1000-steps",
623
+ "jordiclive/gpt4all-alpaca-oa-codealpaca-lora-13b",
624
+ "huggyllama/llama-13b",
625
+ "Voicelab/trurl-2-7b",
626
+ "TFLai/llama-13b-4bit-alpaca",
627
+ "gywy/llama2-13b-chinese-v2",
628
+ "lmsys/longchat-13b-16k",
629
+ "Aspik101/trurl-2-7b-pl-instruct_unload",
630
+ "WizardLM/WizardMath-7B-V1.0",
631
+ "Norquinal/llama-2-7b-claude-chat",
632
+ "TheTravellingEngineer/llama2-7b-chat-hf-dpo",
633
+ "HuggingFaceH4/starchat-beta",
634
+ "joehuangx/spatial-vicuna-7b-v1.5-LoRA",
635
+ "conceptofmind/LLongMA-2-13b-16k",
636
+ "tianyil1/denas-llama2",
637
+ "lmsys/vicuna-7b-v1.3",
638
+ "conceptofmind/LLongMA-2-13b-16k",
639
+ "openchat/opencoderplus",
640
+ "ajibawa-2023/scarlett-7b",
641
+ "dhmeltzer/llama-7b-SFT_eli5_wiki65k_1024_r_64_alpha_16_merged",
642
+ "psyche/kollama2-7b-v2",
643
+ "heegyu/LIMA2-7b-hf",
644
+ "dhmeltzer/llama-7b-SFT-qlora-eli5-wiki_DPO_ds_RM_top_2_1024_r_64_alpha_16",
645
+ "abhishek/llama2guanacotest",
646
+ "jondurbin/airoboros-l2-7b-2.1",
647
+ "llama-anon/instruct-13b",
648
+ "FelixChao/vicuna-7B-physics",
649
+ "Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload",
650
+ "shibing624/chinese-alpaca-plus-13b-hf",
651
+ "davzoku/cria-llama2-7b-v1.3_peft",
652
+ "quantumaikr/llama-2-7b-hf-guanaco-1k",
653
+ "togethercomputer/Llama-2-7B-32K-Instruct",
654
+ "sia-ai/llama-2-7b-1-percent-open-orca-1000-steps-v0",
655
+ "TheTravellingEngineer/llama2-7b-hf-guanaco",
656
+ "Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload",
657
+ "jondurbin/airoboros-l2-7b-gpt4-1.4.1",
658
+ "wahaha1987/llama_7b_sharegpt94k_fastchat",
659
+ "FelixChao/vicuna-7B-chemical",
660
+ "TinyPixel/llama2-7b-oa",
661
+ "chaoyi-wu/MedLLaMA_13B",
662
+ "edor/Platypus2-mini-7B",
663
+ "RoversX/llama-2-7b-hf-small-shards-Samantha-V1-SFT",
664
+ "venkycs/llama-v2-7b-32kC-Security",
665
+ "psyche/kollama2-7b",
666
+ "Fredithefish/Guanaco-7B-Uncensored",
667
+ "TheTravellingEngineer/llama2-7b-chat-hf-guanaco",
668
+ "ehartford/WizardLM-13B-Uncensored",
669
+ "PocketDoc/Dans-CreepingSenseOfDoom",
670
+ "wenge-research/yayi-7b-llama2",
671
+ "georgesung/llama2_7b_chat_uncensored",
672
+ "TinyPixel/llama2-7b-instruct",
673
+ "quantumaikr/QuantumLM-7B",
674
+ "xzuyn/MedicWizard-7B",
675
+ "wenge-research/yayi-7b-llama2",
676
+ "TinyPixel/lima-test",
677
+ "elyza/ELYZA-japanese-Llama-2-7b-instruct",
678
+ "lgaalves/llama-2-7b-hf_open-platypus",
679
+ "ziqingyang/chinese-alpaca-2-7b",
680
+ "TehVenom/Pygmalion-Vicuna-1.1-7b",
681
+ "meta-llama/Llama-2-7b-hf",
682
+ "bongchoi/test-llama2-7b",
683
+ "TaylorAI/Flash-Llama-7B",
684
+ "TheTravellingEngineer/llama2-7b-chat-hf-v2",
685
+ "TheTravellingEngineer/llama2-7b-chat-hf-v4",
686
+ "kashif/stack-llama-2",
687
+ "PeanutJar/LLaMa-2-PeanutButter_v18_A-7B",
688
+ "ToolBench/ToolLLaMA-7b-LoRA",
689
+ "Monero/WizardLM-13b-OpenAssistant-Uncensored",
690
+ "TheTravellingEngineer/llama2-7b-chat-hf-v2",
691
+ "TheTravellingEngineer/llama2-7b-chat-hf-v4",
692
+ "mrm8488/llama-2-coder-7b",
693
+ "elyza/ELYZA-japanese-Llama-2-7b-fast-instruct",
694
+ "clibrain/Llama-2-7b-ft-instruct-es",
695
+ "medalpaca/medalpaca-7b",
696
+ "TheBloke/tulu-7B-fp16",
697
+ "OpenBuddy/openbuddy-openllama-13b-v7-fp16",
698
+ "TaylorAI/FLAN-Llama-7B-2_Llama2-7B-Flash_868_full_model",
699
+ "Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload",
700
+ "jondurbin/airoboros-l2-7b-gpt4-2.0",
701
+ "dhmeltzer/llama-7b-SFT_ds_eli5_1024_r_64_alpha_16_merged",
702
+ "GOAT-AI/GOAT-7B-Community",
703
+ "AtomEchoAI/AtomGPT_56k",
704
+ "julianweng/Llama-2-7b-chat-orcah",
705
+ "TehVenom/Pygmalion-13b-Merged",
706
+ "jondurbin/airoboros-7b-gpt4-1.1",
707
+ "dhmeltzer/llama-7b-SFT_ds_wiki65k_1024_r_64_alpha_16_merged",
708
+ "bofenghuang/vigogne-7b-chat",
709
+ "lmsys/longchat-7b-v1.5-32k",
710
+ "jondurbin/airoboros-l2-7b-gpt4-m2.0",
711
+ "synapsoft/Llama-2-7b-chat-hf-flan2022-1.2M",
712
+ "jondurbin/airoboros-7b-gpt4-1.4",
713
+ "Charlie911/vicuna-7b-v1.5-lora-mctaco",
714
+ "yihan6324/instructmining-platypus-15k",
715
+ "meta-llama/Llama-2-7b-hf",
716
+ "TheTravellingEngineer/llama2-7b-chat-hf-v3",
717
+ "quantumaikr/KoreanLM-hf",
718
+ "openthaigpt/openthaigpt-1.0.0-alpha-7b-chat-ckpt-hf",
719
+ "TheBloke/Llama-2-7B-GPTQ",
720
+ "TheBloke/Llama-2-7B-GPTQ",
721
+ "LLMs/AlpacaGPT4-7B-elina",
722
+ "ehartford/Wizard-Vicuna-7B-Uncensored",
723
+ "TheBloke/Wizard-Vicuna-7B-Uncensored-HF",
724
+ "TheTravellingEngineer/llama2-7b-chat-hf-v3",
725
+ "golaxy/gowizardlm",
726
+ "ehartford/dolphin-llama2-7b",
727
+ "CHIH-HUNG/llama-2-7b-dolphin_10w-test",
728
+ "mncai/chatdoctor",
729
+ "psyche/kollama2-7b-v3",
730
+ "jondurbin/airoboros-7b-gpt4",
731
+ "jondurbin/airoboros-7b",
732
+ "TheBloke/airoboros-7b-gpt4-fp16",
733
+ "mosaicml/mpt-7b-8k-chat",
734
+ "elyza/ELYZA-japanese-Llama-2-7b",
735
+ "bofenghuang/vigogne-7b-instruct",
736
+ "jxhong/CAlign-alpaca-7b",
737
+ "golaxy/goims",
738
+ "jondurbin/airoboros-7b-gpt4-1.2",
739
+ "jphme/orca_mini_v2_ger_7b",
740
+ "psmathur/orca_mini_v2_7b",
741
+ "notstoic/PygmalionCoT-7b",
742
+ "golaxy/gogpt2-13b",
743
+ "golaxy/gogpt2-13b-chat",
744
+ "togethercomputer/LLaMA-2-7B-32K",
745
+ "TheBloke/wizardLM-7B-HF",
746
+ "keyfan/vicuna-chinese-replication-v1.1",
747
+ "golaxy/gogpt2-7b",
748
+ "aiplanet/effi-7b",
749
+ "arver/llama7b-qlora",
750
+ "titan087/OpenLlama13B-Guanaco",
751
+ "chavinlo/alpaca-native",
752
+ "project-baize/baize-healthcare-lora-7B",
753
+ "AlpinDale/pygmalion-instruct",
754
+ "openlm-research/open_llama_13b",
755
+ "jondurbin/airoboros-7b-gpt4-1.3",
756
+ "elyza/ELYZA-japanese-Llama-2-7b-fast",
757
+ "jondurbin/airoboros-gpt-3.5-turbo-100k-7b",
758
+ "uukuguy/speechless-codellama-orca-13b",
759
+ "bigcode/starcoderplus",
760
+ "TheBloke/guanaco-7B-HF",
761
+ "Neko-Institute-of-Science/metharme-7b",
762
+ "TigerResearch/tigerbot-7b-base",
763
+ "golaxy/gogpt-7b",
764
+ "togethercomputer/LLaMA-2-7B-32K",
765
+ "yhyhy3/open_llama_7b_v2_med_instruct",
766
+ "ajibawa-2023/carl-7b",
767
+ "stabilityai/stablelm-base-alpha-7b-v2",
768
+ "conceptofmind/LLongMA-2-7b-16k",
769
+ "TehVenom/Pygmalion_AlpacaLora-7b",
770
+ "jondurbin/airoboros-7b-gpt4-1.4.1-qlora",
771
+ "wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard",
772
+ "ausboss/llama7b-wizardlm-unfiltered",
773
+ "project-baize/baize-v2-7b",
774
+ "LMFlow/Robin-v2",
775
+ "HanningZhang/Robin-v2",
776
+ "LMFlow/Robin-7b-v2",
777
+ "OptimalScale/robin-7b-v2-delta",
778
+ "uukuguy/speechless-codellama-platypus-13b",
779
+ "jerryjalapeno/nart-100k-7b",
780
+ "wenge-research/yayi-13b-llama2",
781
+ "fireballoon/baichuan-vicuna-chinese-7b",
782
+ "jlevin/guanaco-unchained-llama-2-7b",
783
+ "csitfun/llama-7b-logicot",
784
+ "DevaMalla/llama7b_alpaca_1gpu_bf16",
785
+ "WeOpenML/PandaLM-Alpaca-7B-v1",
786
+ "illuin/test-custom-llama",
787
+ "yeontaek/WizardCoder-Python-13B-LoRa",
788
+ "ashercn97/giraffe-7b",
789
+ "mosaicml/mpt-7b-chat",
790
+ "abhishek/autotrain-llama-alpaca-peft-52508123785",
791
+ "Neko-Institute-of-Science/pygmalion-7b",
792
+ "TFLai/llama-7b-4bit-alpaca",
793
+ "huggingface/llama-7b",
794
+ "TheBloke/Planner-7B-fp16",
795
+ "shibing624/chinese-llama-plus-13b-hf",
796
+ "AGI-inc/lora_moe_7b_baseline",
797
+ "DevaMalla/llama-base-7b",
798
+ "AGI-inc/lora_moe_7b",
799
+ "togethercomputer/GPT-JT-6B-v0",
800
+ "ehartford/WizardLM-7B-Uncensored",
801
+ "shibing624/chinese-alpaca-plus-7b-hf",
802
+ "beomi/llama-2-ko-7b",
803
+ "mosaicml/mpt-7b-8k-instruct",
804
+ "Enno-Ai/ennodata-7b",
805
+ "mosaicml/mpt-7b-instruct",
806
+ "facebook/opt-iml-max-30b",
807
+ "WeOpenML/Alpaca-7B-v1",
808
+ "TheBloke/Project-Baize-v2-7B-GPTQ",
809
+ "codellama/CodeLlama-13b-Instruct-hf",
810
+ "TheBloke/CodeLlama-13B-Instruct-fp16",
811
+ "facebook/galactica-30b",
812
+ "FreedomIntelligence/phoenix-inst-chat-7b",
813
+ "openlm-research/open_llama_7b_v2",
814
+ "GeorgiaTechResearchInstitute/galpaca-30b",
815
+ "THUDM/chatglm2-6b",
816
+ "togethercomputer/GPT-JT-6B-v1",
817
+ "TheBloke/koala-7B-HF",
818
+ "nathan0/mpt_delta_tuned_model_v3",
819
+ "nathan0/mpt_delta_tuned_model_v2",
820
+ "GeorgiaTechResearchInstitute/galpaca-30b",
821
+ "JosephusCheung/Guanaco",
822
+ "shareAI/CodeLLaMA-chat-13b-Chinese",
823
+ "TigerResearch/tigerbot-7b-sft",
824
+ "Writer/InstructPalmyra-20b",
825
+ "OpenAssistant/codellama-13b-oasst-sft-v10",
826
+ "bigscience/bloomz-7b1-mt",
827
+ "nathan0/mpt_delta_tuned_model_v3",
828
+ "VMware/open-llama-7b-open-instruct",
829
+ "baichuan-inc/Baichuan-7B",
830
+ "anas-awadalla/mpt-7b",
831
+ "mosaicml/mpt-7b",
832
+ "bigscience/bloomz-7b1",
833
+ "ziqingyang/chinese-llama-2-7b",
834
+ "OpenAssistant/codellama-13b-oasst-sft-v10",
835
+ "wenge-research/yayi-7b",
836
+ "tiiuae/falcon-7b",
837
+ "togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1",
838
+ "togethercomputer/RedPajama-INCITE-7B-Instruct",
839
+ "TheBloke/landmark-attention-llama7b-fp16",
840
+ "togethercomputer/GPT-JT-Moderation-6B",
841
+ "h2oai/h2ogpt-gm-oasst1-en-1024-20b",
842
+ "dvruette/gpt-neox-20b-full-precision",
843
+ "TehVenom/Moderator-Chan_GPT-JT-6b",
844
+ "dvruette/oasst-gpt-neox-20b-1000-steps",
845
+ "AlekseyKorshuk/pygmalion-6b-vicuna-chatml",
846
+ "facebook/opt-66b",
847
+ "Salesforce/codegen-16B-nl",
848
+ "Vmware/open-llama-7b-v2-open-instruct",
849
+ "mosaicml/mpt-7b-storywriter",
850
+ "acrastt/Marx-3B-V2",
851
+ "openlm-research/open_llama_7b",
852
+ "Fredithefish/ReasonixPajama-3B-HF",
853
+ "togethercomputer/GPT-NeoXT-Chat-Base-20B",
854
+ "psmathur/orca_mini_13b",
855
+ "RWKV/rwkv-raven-14b",
856
+ "h2oai/h2ogpt-oasst1-512-20b",
857
+ "acrastt/Marx-3B",
858
+ "klosax/open_llama_13b_600bt_preview",
859
+ "synapsoft/Llama-2-7b-hf-flan2022-1.2M",
860
+ "OpenAssistant/oasst-sft-1-pythia-12b",
861
+ "golaxy/gogpt-7b-bloom",
862
+ "Writer/palmyra-large",
863
+ "psmathur/orca_mini_7b",
864
+ "dvruette/oasst-pythia-12b-6000-steps",
865
+ "NousResearch/CodeLlama-13b-hf",
866
+ "codellama/CodeLlama-13b-hf",
867
+ "h2oai/h2ogpt-gm-oasst1-multilang-1024-20b",
868
+ "VMware/open-llama-0.7T-7B-open-instruct-v1.1",
869
+ "dvruette/oasst-pythia-12b-flash-attn-5000-steps",
870
+ "dvruette/oasst-gpt-neox-20b-3000-steps",
871
+ "RobbeD/OpenLlama-Platypus-3B",
872
+ "facebook/opt-30b",
873
+ "acrastt/Puma-3B",
874
+ "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
875
+ "dvruette/oasst-pythia-12b-pretrained-sft",
876
+ "digitous/GPT-R",
877
+ "acrastt/Griffin-3B",
878
+ "togethercomputer/RedPajama-INCITE-Base-7B-v0.1",
879
+ "togethercomputer/RedPajama-INCITE-7B-Base",
880
+ "CobraMamba/mamba-gpt-3b-v3",
881
+ "Danielbrdz/CodeBarcenas-7b",
882
+ "l3utterfly/open-llama-3b-v2-layla",
883
+ "CobraMamba/mamba-gpt-3b-v2",
884
+ "OpenAssistant/pythia-12b-sft-v8-7k-steps",
885
+ "KoboldAI/GPT-NeoX-20B-Erebus",
886
+ "RobbeD/Orca-Platypus-3B",
887
+ "h2oai/h2ogpt-gm-oasst1-en-1024-12b",
888
+ "OpenAssistant/pythia-12b-sft-v8-2.5k-steps",
889
+ "AlekseyKorshuk/chatml-pyg-v1",
890
+ "togethercomputer/RedPajama-INCITE-Chat-7B-v0.1",
891
+ "togethercomputer/RedPajama-INCITE-7B-Chat",
892
+ "digitous/Javelin-R",
893
+ "dvruette/oasst-pythia-12b-reference",
894
+ "EleutherAI/gpt-neox-20b",
895
+ "KoboldAI/fairseq-dense-13B",
896
+ "OpenAssistant/pythia-12b-sft-v8-rlhf-2k-steps",
897
+ "codellama/CodeLlama-7b-Instruct-hf",
898
+ "digitous/Javelin-GPTJ",
899
+ "KoboldAI/GPT-NeoX-20B-Skein",
900
+ "digitous/Javalion-R",
901
+ "h2oai/h2ogpt-oasst1-512-12b",
902
+ "acrastt/Bean-3B",
903
+ "KoboldAI/GPT-J-6B-Skein",
904
+ "nomic-ai/gpt4all-j",
905
+ "databricks/dolly-v2-12b",
906
+ "TehVenom/Dolly_Shygmalion-6b-Dev_V8P2",
907
+ "databricks/dolly-v2-7b",
908
+ "Aspik101/WizardVicuna-Uncensored-3B-instruct-PL-lora_unload",
909
+ "digitous/Adventien-GPTJ",
910
+ "openlm-research/open_llama_3b_v2",
911
+ "RWKV/rwkv-4-14b-pile",
912
+ "Lazycuber/Janemalion-6B",
913
+ "OpenAssistant/pythia-12b-pre-v8-12.5k-steps",
914
+ "digitous/Janin-R",
915
+ "kfkas/Llama-2-ko-7b-Chat",
916
+ "heegyu/WizardVicuna-Uncensored-3B-0719",
917
+ "h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt",
918
+ "TaylorAI/Flash-Llama-3B",
919
+ "kfkas/Llama-2-ko-7b-Chat",
920
+ "digitous/Skegma-GPTJ",
921
+ "digitous/Javalion-GPTJ",
922
+ "Pirr/pythia-13b-deduped-green_devil",
923
+ "TehVenom/PPO_Shygmalion-V8p4_Dev-6b",
924
+ "dvruette/oasst-pythia-6.9b-4000-steps",
925
+ "heegyu/WizardVicuna-3B-0719",
926
+ "psmathur/orca_mini_3b",
927
+ "OpenAssistant/galactica-6.7b-finetuned",
928
+ "frank098/orca_mini_3b_juniper",
929
+ "PygmalionAI/pygmalion-6b",
930
+ "TehVenom/PPO_Pygway-V8p4_Dev-6b",
931
+ "TFLai/gpt-neox-20b-4bit-alpaca",
932
+ "Corianas/gpt-j-6B-Dolly",
933
+ "TehVenom/Dolly_Shygmalion-6b",
934
+ "digitous/Janin-GPTJ",
935
+ "TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4",
936
+ "EleutherAI/gpt-j-6b",
937
+ "KoboldAI/GPT-J-6B-Shinen",
938
+ "TehVenom/Dolly_Malion-6b",
939
+ "TehVenom/ChanMalion",
940
+ "Salesforce/codegen-6B-nl",
941
+ "Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4",
942
+ "KoboldAI/GPT-J-6B-Janeway",
943
+ "togethercomputer/RedPajama-INCITE-Chat-3B-v1",
944
+ "togethercomputer/Pythia-Chat-Base-7B",
945
+ "heegyu/RedTulu-Uncensored-3B-0719",
946
+ "KoboldAI/PPO_Pygway-6b-Mix",
947
+ "KoboldAI/OPT-13B-Erebus",
948
+ "KoboldAI/fairseq-dense-6.7B",
949
+ "EleutherAI/pythia-12b-deduped",
950
+ "pszemraj/pythia-6.9b-HC3",
951
+ "Fredithefish/Guanaco-3B-Uncensored-v2",
952
+ "facebook/opt-13b",
953
+ "TehVenom/GPT-J-Pyg_PPO-6B",
954
+ "EleutherAI/pythia-6.9b-deduped",
955
+ "Devio/test-1400",
956
+ "Fredithefish/Guanaco-3B-Uncensored",
957
+ "codellama/CodeLlama-7b-hf",
958
+ "acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1",
959
+ "Fredithefish/ScarletPajama-3B-HF",
960
+ "KoboldAI/OPT-13B-Nerybus-Mix",
961
+ "YeungNLP/firefly-bloom-7b1",
962
+ "DanielSc4/RedPajama-INCITE-Chat-3B-v1-RL-LoRA-8bit-test1",
963
+ "klosax/open_llama_7b_400bt_preview",
964
+ "KoboldAI/OPT-13B-Nerys-v2",
965
+ "TehVenom/PPO_Shygmalion-6b",
966
+ "amazon/LightGPT",
967
+ "KnutJaegersberg/black_goo_recipe_c",
968
+ "NousResearch/CodeLlama-7b-hf",
969
+ "togethercomputer/RedPajama-INCITE-Instruct-3B-v1",
970
+ "heegyu/WizardVicuna-open-llama-3b-v2",
971
+ "bigscience/bloom-7b1",
972
+ "Devio/test-22B",
973
+ "RWKV/rwkv-raven-7b",
974
+ "hakurei/instruct-12b",
975
+ "CobraMamba/mamba-gpt-3b",
976
+ "KnutJaegersberg/black_goo_recipe_a",
977
+ "acrastt/OmegLLaMA-3B",
978
+ "codellama/CodeLlama-7b-Instruct-hf",
979
+ "h2oai/h2ogpt-oig-oasst1-512-6_9b",
980
+ "KoboldAI/OPT-6.7B-Erebus",
981
+ "facebook/opt-6.7b",
982
+ "KnutJaegersberg/black_goo_recipe_d",
983
+ "KnutJaegersberg/LLongMA-3b-LIMA",
984
+ "KnutJaegersberg/black_goo_recipe_b",
985
+ "KoboldAI/OPT-6.7B-Nerybus-Mix",
986
+ "health360/Healix-3B",
987
+ "EleutherAI/pythia-12b",
988
+ "Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K",
989
+ "GeorgiaTechResearchInstitute/galactica-6.7b-evol-instruct-70k",
990
+ "h2oai/h2ogpt-oig-oasst1-256-6_9b",
991
+ "ikala/bloom-zh-3b-chat",
992
+ "Taekyoon/llama2-ko-7b-test",
993
+ "anhnv125/pygmalion-6b-roleplay",
994
+ "TehVenom/DiffMerge_Pygmalion_Main-onto-V8P4",
995
+ "KoboldAI/OPT-6B-nerys-v2",
996
+ "Lazycuber/pyg-instruct-wizardlm",
997
+ "Devio/testC",
998
+ "KoboldAI/OPT-30B-Erebus",
999
+ "Fredithefish/CrimsonPajama",
1000
+ "togethercomputer/RedPajama-INCITE-Base-3B-v1",
1001
+ "bigscience/bloomz-3b",
1002
+ "conceptofmind/Open-LLongMA-3b",
1003
+ "RWKV/rwkv-4-7b-pile",
1004
+ "openlm-research/open_llama_3b",
1005
+ "ewof/koishi-instruct-3b",
1006
+ "DanielSc4/RedPajama-INCITE-Chat-3B-v1-FT-LoRA-8bit-test1",
1007
+ "cerebras/Cerebras-GPT-13B",
1008
+ "EleutherAI/pythia-6.7b",
1009
+ "aisquared/chopt-2_7b",
1010
+ "Azure99/blossom-v1-3b",
1011
+ "PSanni/Deer-3b",
1012
+ "bertin-project/bertin-gpt-j-6B-alpaca",
1013
+ "OpenBuddy/openbuddy-openllama-3b-v10-bf16",
1014
+ "KoboldAI/fairseq-dense-2.7B",
1015
+ "ehartford/CodeLlama-34b-Instruct-hf",
1016
+ "codellama/CodeLlama-34b-Instruct-hf",
1017
+ "TheBloke/CodeLlama-34B-Instruct-fp16",
1018
+ "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2",
1019
+ "openlm-research/open_llama_7b_700bt_preview",
1020
+ "NbAiLab/nb-gpt-j-6B-alpaca",
1021
+ "KoboldAI/OPT-2.7B-Erebus",
1022
+ "Writer/camel-5b-hf",
1023
+ "EleutherAI/pythia-2.7b",
1024
+ "facebook/xglm-7.5B",
1025
+ "EleutherAI/pythia-2.8b-deduped",
1026
+ "klosax/open_llama_3b_350bt_preview",
1027
+ "klosax/openllama-3b-350bt",
1028
+ "KoboldAI/OPT-2.7B-Nerybus-Mix",
1029
+ "KoboldAI/GPT-J-6B-Adventure",
1030
+ "cerebras/Cerebras-GPT-6.7B",
1031
+ "TFLai/pythia-2.8b-4bit-alpaca",
1032
+ "facebook/opt-2.7b",
1033
+ "KoboldAI/OPT-2.7B-Nerys-v2",
1034
+ "bigscience/bloom-3b",
1035
+ "Devio/test100",
1036
+ "RWKV/rwkv-raven-3b",
1037
+ "Azure99/blossom-v2-3b",
1038
+ "codellama/CodeLlama-34b-Python-hf",
1039
+ "bhenrym14/airoboros-33b-gpt4-1.4.1-PI-8192-fp16",
1040
+ "EleutherAI/gpt-neo-2.7B",
1041
+ "danielhanchen/open_llama_3b_600bt_preview",
1042
+ "HuggingFaceH4/starchat-alpha",
1043
+ "pythainlp/wangchanglm-7.5B-sft-en-sharded",
1044
+ "beaugogh/pythia-1.4b-deduped-sharegpt",
1045
+ "HWERI/pythia-1.4b-deduped-sharegpt",
1046
+ "OpenAssistant/stablelm-7b-sft-v7-epoch-3",
1047
+ "codellama/CodeLlama-7b-Python-hf",
1048
+ "aisquared/chopt-1_3b",
1049
+ "PygmalionAI/metharme-1.3b",
1050
+ "Linly-AI/Chinese-LLaMA-2-13B-hf",
1051
+ "chargoddard/llama-2-34b-uncode",
1052
+ "RWKV/rwkv-4-3b-pile",
1053
+ "pythainlp/wangchanglm-7.5B-sft-enth",
1054
+ "MBZUAI/LaMini-GPT-1.5B",
1055
+ "Writer/palmyra-base",
1056
+ "KoboldAI/fairseq-dense-1.3B",
1057
+ "EleutherAI/pythia-1.4b-deduped",
1058
+ "MBZUAI/lamini-neo-1.3b",
1059
+ "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt",
1060
+ "sartmis1/starcoder-finetune-openapi",
1061
+ "MayaPH/opt-flan-iml-6.7b",
1062
+ "facebook/xglm-4.5B",
1063
+ "WizardLM/WizardCoder-15B-V1.0",
1064
+ "facebook/opt-iml-max-1.3b",
1065
+ "stabilityai/stablelm-tuned-alpha-7b",
1066
+ "aisquared/dlite-v2-1_5b",
1067
+ "stabilityai/stablelm-base-alpha-7b",
1068
+ "sartmis1/starcoder-finetune-selfinstruct",
1069
+ "lizhuang144/starcoder_mirror",
1070
+ "bigcode/starcoder",
1071
+ "TheBloke/CodeLlama-34B-Python-fp16",
1072
+ "open-llm-leaderboard/bloomz-1b7-4bit-alpaca-auto-eval-adapter-applied",
1073
+ "ehartford/CodeLlama-34b-Python-hf",
1074
+ "codellama/CodeLlama-7b-Python-hf",
1075
+ "GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct",
1076
+ "LoupGarou/WizardCoder-Guanaco-15B-V1.0",
1077
+ "golaxy/gogpt-3b-bloom",
1078
+ "EleutherAI/pythia-1.3b",
1079
+ "codellama/CodeLlama-13b-Python-hf",
1080
+ "hakurei/lotus-12B",
1081
+ "NYTK/PULI-GPTrio",
1082
+ "facebook/opt-1.3b",
1083
+ "TheBloke/CodeLlama-13B-Python-fp16",
1084
+ "codellama/CodeLlama-13b-Python-hf",
1085
+ "RWKV/rwkv-raven-1b5",
1086
+ "PygmalionAI/pygmalion-2.7b",
1087
+ "bigscience/bloom-1b7",
1088
+ "gpt2-xl",
1089
+ "LoupGarou/WizardCoder-Guanaco-15B-V1.1",
1090
+ "RWKV/rwkv-4-1b5-pile",
1091
+ "codellama/CodeLlama-34b-hf",
1092
+ "NousResearch/CodeLlama-34b-hf",
1093
+ "rinna/bilingual-gpt-neox-4b-8k",
1094
+ "lxe/Cerebras-GPT-2.7B-Alpaca-SP",
1095
+ "cerebras/Cerebras-GPT-2.7B",
1096
+ "jzjiao/opt-1.3b-rlhf",
1097
+ "EleutherAI/gpt-neo-1.3B",
1098
+ "aisquared/dlite-v1-1_5b",
1099
+ "Corianas/Quokka_2.7b",
1100
+ "MrNJK/gpt2-xl-sft",
1101
+ "facebook/galactica-1.3b",
1102
+ "aisquared/dlite-v2-774m",
1103
+ "EleutherAI/pythia-1b-deduped",
1104
+ "Kunhao/pile-7b-250b-tokens",
1105
+ "w601sxs/b1ade-1b",
1106
+ "rinna/bilingual-gpt-neox-4b",
1107
+ "shaohang/SparseOPT-1.3B",
1108
+ "shaohang/Sparse0.5_OPT-1.3",
1109
+ "EleutherAI/polyglot-ko-12.8b",
1110
+ "Salesforce/codegen-6B-multi",
1111
+ "bigscience/bloom-1b1",
1112
+ "TFLai/gpt-neo-1.3B-4bit-alpaca",
1113
+ "FabbriSimo01/Bloom_1b_Quantized",
1114
+ "MBZUAI/LaMini-GPT-774M",
1115
+ "Locutusque/gpt2-large-conversational",
1116
+ "Devio/test-3b",
1117
+ "stabilityai/stablelm-tuned-alpha-3b",
1118
+ "PygmalionAI/pygmalion-1.3b",
1119
+ "KoboldAI/fairseq-dense-355M",
1120
+ "Rachneet/gpt2-xl-alpaca",
1121
+ "gpt2-large",
1122
+ "Mikivis/gpt2-large-lora-sft",
1123
+ "stabilityai/stablelm-base-alpha-3b",
1124
+ "gpt2-medium",
1125
+ "Kunhao/pile-7b",
1126
+ "aisquared/dlite-v1-774m",
1127
+ "aisquared/dlite-v2-355m",
1128
+ "YeungNLP/firefly-bloom-2b6-v2",
1129
+ "KnutJaegersberg/gpt-2-xl-EvolInstruct",
1130
+ "KnutJaegersberg/galactica-orca-wizardlm-1.3b",
1131
+ "cerebras/Cerebras-GPT-1.3B",
1132
+ "FabbriSimo01/Cerebras_1.3b_Quantized",
1133
+ "facebook/xglm-1.7B",
1134
+ "EleutherAI/pythia-410m-deduped",
1135
+ "TheBloke/GPlatty-30B-SuperHOT-8K-fp16",
1136
+ "DataLinguistic/DataLinguistic-34B-V1.0",
1137
+ "Corianas/Quokka_1.3b",
1138
+ "TheTravellingEngineer/bloom-560m-RLHF-v2",
1139
+ "Corianas/1.3b",
1140
+ "RWKV/rwkv-4-430m-pile",
1141
+ "porkorbeef/Llama-2-13b-sf",
1142
+ "xhyi/PT_GPTNEO350_ATG",
1143
+ "TheBloke/Wizard-Vicuna-13B-Uncensored-GPTQ",
1144
+ "bigscience/bloomz-560m",
1145
+ "TheBloke/medalpaca-13B-GPTQ-4bit",
1146
+ "TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16",
1147
+ "aisquared/dlite-v1-355m",
1148
+ "uukuguy/speechless-codellama-orca-airoboros-13b-0.10e",
1149
+ "yhyhy3/med-orca-instruct-33b",
1150
+ "TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16",
1151
+ "TheTravellingEngineer/bloom-1b1-RLHF",
1152
+ "MBZUAI/lamini-cerebras-1.3b",
1153
+ "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1",
1154
+ "TheBloke/WizardLM-7B-uncensored-GPTQ",
1155
+ "TheBloke/EverythingLM-13B-16K-GPTQ",
1156
+ "quantumaikr/open_llama_7b_hf",
1157
+ "TheBloke/chronos-wizardlm-uc-scot-st-13B-GPTQ",
1158
+ "TheBloke/WizardLM-30B-Uncensored-GPTQ",
1159
+ "IDEA-CCNL/Ziya-LLaMA-13B-v1",
1160
+ "Phind/Phind-CodeLlama-34B-v1",
1161
+ "robowaifudev/megatron-gpt2-345m",
1162
+ "MayaPH/GodziLLa-30B-instruct",
1163
+ "TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16",
1164
+ "uukuguy/speechless-codellama-orca-platypus-13b-0.10e",
1165
+ "doas/test2",
1166
+ "BreadAi/PM_modelV2",
1167
+ "bigcode/santacoder",
1168
+ "TheBloke/wizard-vicuna-13B-GPTQ",
1169
+ "porkorbeef/Llama-2-13b",
1170
+ "TehVenom/DiffMerge-DollyGPT-Pygmalion",
1171
+ "PygmalionAI/pygmalion-350m",
1172
+ "TheBloke/orca_mini_v3_7B-GPTQ",
1173
+ "TheBloke/WizardLM-Uncensored-SuperCOT-StoryTelling-30B-GPTQ",
1174
+ "TheBloke/WizardLM-30B-GPTQ",
1175
+ "bigscience/bloom-560m",
1176
+ "TFLai/gpt2-turkish-uncased",
1177
+ "TheBloke/guanaco-33B-GPTQ",
1178
+ "TheBloke/openchat_v2_openorca_preview-GPTQ",
1179
+ "porkorbeef/Llama-2-13b-public",
1180
+ "TheBloke/LongChat-13B-GPTQ",
1181
+ "yhyhy3/med-orca-instruct-33b",
1182
+ "TheBloke/airoboros-33B-gpt4-1-4-SuperHOT-8K-fp16",
1183
+ "TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16",
1184
+ "MayaPH/FinOPT-Franklin",
1185
+ "TheBloke/WizardLM-33B-V1.0-Uncensored-GPTQ",
1186
+ "TheBloke/Project-Baize-v2-13B-GPTQ",
1187
+ "malhajar/Platypus2-70B-instruct-4bit-gptq",
1188
+ "KoboldAI/OPT-350M-Erebus",
1189
+ "rishiraj/bloom-560m-guanaco",
1190
+ "Panchovix/WizardLM-33B-V1.0-Uncensored-SuperHOT-8k",
1191
+ "doas/test5",
1192
+ "vicgalle/alpaca-7b",
1193
+ "beomi/KoAlpaca-Polyglot-5.8B",
1194
+ "Phind/Phind-CodeLlama-34B-Python-v1",
1195
+ "timdettmers/guanaco-65b-merged",
1196
+ "TheBloke/wizard-mega-13B-GPTQ",
1197
+ "MayaPH/GodziLLa-30B-plus",
1198
+ "TheBloke/Platypus-30B-SuperHOT-8K-fp16",
1199
+ "facebook/opt-350m",
1200
+ "KoboldAI/OPT-350M-Nerys-v2",
1201
+ "TheBloke/robin-33B-v2-GPTQ",
1202
+ "jaspercatapang/Echidna-30B",
1203
+ "TheBloke/llama-30b-supercot-SuperHOT-8K-fp16",
1204
+ "marcchew/test1",
1205
+ "Harshvir/LaMini-Neo-1.3B-Mental-Health_lora",
1206
+ "golaxy/gogpt-560m",
1207
+ "TheBloke/orca_mini_13B-GPTQ",
1208
+ "Panchovix/airoboros-33b-gpt4-1.2-SuperHOT-8k",
1209
+ "Aspik101/tulu-7b-instruct-pl-lora_unload",
1210
+ "Phind/Phind-CodeLlama-34B-v2",
1211
+ "BreadAi/MusePy-1-2",
1212
+ "cerebras/Cerebras-GPT-590M",
1213
+ "microsoft/CodeGPT-small-py",
1214
+ "victor123/WizardLM-13B-1.0",
1215
+ "OptimalScale/robin-65b-v2-delta",
1216
+ "voidful/changpt-bart",
1217
+ "FabbriSimo01/GPT_Large_Quantized",
1218
+ "MayaPH/FinOPT-Lincoln",
1219
+ "KoboldAI/fairseq-dense-125M",
1220
+ "SebastianSchramm/Cerebras-GPT-111M-instruction",
1221
+ "TheTravellingEngineer/bloom-560m-RLHF",
1222
+ "breadlicker45/dough-instruct-base-001",
1223
+ "WizardLM/WizardLM-30B-V1.0",
1224
+ "WizardLM/WizardLM-30B-V1.0",
1225
+ "WizardLM/WizardLM-30B-V1.0",
1226
+ "TaylorAI/Flash-Llama-30M-20001",
1227
+ "porkorbeef/Llama-2-13b-12_153950",
1228
+ "huggingtweets/bladeecity-jerma985",
1229
+ "KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct",
1230
+ "bhenrym14/airoboros-33b-gpt4-1.4.1-lxctx-PI-16384-fp16",
1231
+ "microsoft/DialoGPT-small",
1232
+ "Corianas/590m",
1233
+ "facebook/xglm-564M",
1234
+ "EleutherAI/gpt-neo-125m",
1235
+ "EleutherAI/pythia-160m-deduped",
1236
+ "klosax/pythia-160m-deduped-step92k-193bt",
1237
+ "MBZUAI/lamini-neo-125m",
1238
+ "bigcode/tiny_starcoder_py",
1239
+ "concedo/OPT-19M-ChatSalad",
1240
+ "anton-l/gpt-j-tiny-random",
1241
+ "grantprice/Cerebras-GPT-590M-finetuned-DND",
1242
+ "deepnight-research/zsc-text",
1243
+ "WangZeJun/bloom-820m-chat",
1244
+ "cerebras/Cerebras-GPT-256M",
1245
+ "ai-forever/rugpt3large_based_on_gpt2",
1246
+ "alibidaran/medical_transcription_generator",
1247
+ "Deci/DeciCoder-1b",
1248
+ "microsoft/DialoGPT-medium",
1249
+ "ogimgio/gpt-neo-125m-neurallinguisticpioneers",
1250
+ "open-llm-leaderboard/bloom-560m-4bit-alpaca-auto-eval-adapter-applied",
1251
+ "BreadAi/gpt-YA-1-1_160M",
1252
+ "microsoft/DialoGPT-large",
1253
+ "facebook/opt-125m",
1254
+ "huggingtweets/jerma985",
1255
+ "Locutusque/gpt2-conversational-or-qa",
1256
+ "concedo/Pythia-70M-ChatSalad",
1257
+ "roneneldan/TinyStories-1M",
1258
+ "BreadAi/DiscordPy",
1259
+ "bigcode/gpt_bigcode-santacoder",
1260
+ "Tincando/fiction_story_generator",
1261
+ "klosax/pythia-70m-deduped-step44k-92bt",
1262
+ "Quake24/easyTermsSummerizer",
1263
+ "BreadAi/gpt-YA-1-1_70M",
1264
+ "EleutherAI/pythia-160m",
1265
+ "euclaise/gpt-neox-122m-minipile-digits",
1266
+ "MBZUAI/lamini-cerebras-590m",
1267
+ "nicholasKluge/Aira-124M",
1268
+ "MayaPH/FinOPT-Washington",
1269
+ "cyberagent/open-calm-large",
1270
+ "BreadAi/StoryPy",
1271
+ "EleutherAI/pythia-70m",
1272
+ "BreadAi/gpt-Youtube",
1273
+ "roneneldan/TinyStories-33M",
1274
+ "EleutherAI/pythia-70m-deduped",
1275
+ "lgaalves/gpt2_guanaco-dolly-platypus",
1276
+ "Corianas/Quokka_590m",
1277
+ "lgaalves/gpt2_platypus-dolly-guanaco",
1278
+ "cyberagent/open-calm-7b",
1279
+ "RWKV/rwkv-4-169m-pile",
1280
+ "gpt2",
1281
+ "roneneldan/TinyStories-28M",
1282
+ "lgaalves/gpt2_open-platypus",
1283
+ "gpt2",
1284
+ "SaylorTwift/gpt2_test",
1285
+ "roneneldan/TinyStories-3M",
1286
+ "nthngdy/pythia-owt2-70m-50k",
1287
+ "Corianas/256_5epoch",
1288
+ "roneneldan/TinyStories-8M",
1289
+ "lgaalves/gpt2-dolly",
1290
+ "nthngdy/pythia-owt2-70m-100k",
1291
+ "aisquared/dlite-v2-124m",
1292
+ "mncai/SGPT-1.3B-insurance-epoch10",
1293
+ "huggingtweets/gladosystem",
1294
+ "abhiramtirumala/DialoGPT-sarcastic-medium",
1295
+ "MBZUAI/lamini-cerebras-256m",
1296
+ "cerebras/Cerebras-GPT-111M",
1297
+ "uberkie/metharme-1.3b-finetuned",
1298
+ "MBZUAI/lamini-cerebras-111m",
1299
+ "psyche/kogpt",
1300
+ "Corianas/Quokka_256m",
1301
+ "vicgalle/gpt2-alpaca-gpt4",
1302
+ "aisquared/dlite-v1-124m",
1303
+ "Mikivis/xuanxuan",
1304
+ "MBZUAI/LaMini-GPT-124M",
1305
+ "vicgalle/gpt2-alpaca",
1306
+ "huashiyiqike/testmodel",
1307
+ "Corianas/111m",
1308
+ "baseline",
1309
+ ]
src/tools/plots.py ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ import numpy as np
3
+ import plotly.express as px
4
+ from plotly.graph_objs import Figure
5
+
6
+ from src.leaderboard.filter_models import FLAGGED_MODELS
7
+ from src.display.utils import human_baseline_row as HUMAN_BASELINE, AutoEvalColumn, Tasks, Task, BENCHMARK_COLS
8
+ from src.leaderboard.read_evals import EvalResult
9
+
10
+
11
+
12
+ def create_scores_df(raw_data: list[EvalResult]) -> pd.DataFrame:
13
+ """
14
+ Generates a DataFrame containing the maximum scores until each date.
15
+
16
+ :param results_df: A DataFrame containing result information including metric scores and dates.
17
+ :return: A new DataFrame containing the maximum scores until each date for every metric.
18
+ """
19
+ # Step 1: Ensure 'date' is in datetime format and sort the DataFrame by it
20
+ results_df = pd.DataFrame(raw_data)
21
+ #results_df["date"] = pd.to_datetime(results_df["date"], format="mixed", utc=True)
22
+ results_df.sort_values(by="date", inplace=True)
23
+
24
+ # Step 2: Initialize the scores dictionary
25
+ scores = {k: [] for k in BENCHMARK_COLS + [AutoEvalColumn.average.name]}
26
+
27
+ # Step 3: Iterate over the rows of the DataFrame and update the scores dictionary
28
+ for task in [t.value for t in Tasks] + [Task("Average", "avg", AutoEvalColumn.average.name)]:
29
+ current_max = 0
30
+ last_date = ""
31
+ column = task.col_name
32
+ for _, row in results_df.iterrows():
33
+ current_model = row["full_model"]
34
+ if current_model in FLAGGED_MODELS:
35
+ continue
36
+
37
+ current_date = row["date"]
38
+ if task.benchmark == "Average":
39
+ current_score = np.mean(list(row["results"].values()))
40
+ else:
41
+ current_score = row["results"][task.benchmark]
42
+
43
+ if current_score > current_max:
44
+ if current_date == last_date and len(scores[column]) > 0:
45
+ scores[column][-1] = {"model": current_model, "date": current_date, "score": current_score}
46
+ else:
47
+ scores[column].append({"model": current_model, "date": current_date, "score": current_score})
48
+ current_max = current_score
49
+ last_date = current_date
50
+
51
+ # Step 4: Return all dictionaries as DataFrames
52
+ return {k: pd.DataFrame(v) for k, v in scores.items()}
53
+
54
+
55
+ def create_plot_df(scores_df: dict[str: pd.DataFrame]) -> pd.DataFrame:
56
+ """
57
+ Transforms the scores DataFrame into a new format suitable for plotting.
58
+
59
+ :param scores_df: A DataFrame containing metric scores and dates.
60
+ :return: A new DataFrame reshaped for plotting purposes.
61
+ """
62
+ # Initialize the list to store DataFrames
63
+ dfs = []
64
+
65
+ # Iterate over the cols and create a new DataFrame for each column
66
+ for col in BENCHMARK_COLS + [AutoEvalColumn.average.name]:
67
+ d = scores_df[col].reset_index(drop=True)
68
+ d["task"] = col
69
+ dfs.append(d)
70
+
71
+ # Concatenate all the created DataFrames
72
+ concat_df = pd.concat(dfs, ignore_index=True)
73
+
74
+ # Sort values by 'date'
75
+ concat_df.sort_values(by="date", inplace=True)
76
+ concat_df.reset_index(drop=True, inplace=True)
77
+ return concat_df
78
+
79
+
80
+ def create_metric_plot_obj(
81
+ df: pd.DataFrame, metrics: list[str], title: str
82
+ ) -> Figure:
83
+ """
84
+ Create a Plotly figure object with lines representing different metrics
85
+ and horizontal dotted lines representing human baselines.
86
+
87
+ :param df: The DataFrame containing the metric values, names, and dates.
88
+ :param metrics: A list of strings representing the names of the metrics
89
+ to be included in the plot.
90
+ :param title: A string representing the title of the plot.
91
+ :return: A Plotly figure object with lines representing metrics and
92
+ horizontal dotted lines representing human baselines.
93
+ """
94
+
95
+ # Filter the DataFrame based on the specified metrics
96
+ df = df[df["task"].isin(metrics)]
97
+
98
+ # Filter the human baselines based on the specified metrics
99
+ filtered_human_baselines = {k: v for k, v in HUMAN_BASELINE.items() if k in metrics}
100
+
101
+ # Create a line figure using plotly express with specified markers and custom data
102
+ fig = px.line(
103
+ df,
104
+ x="date",
105
+ y="score",
106
+ color="task",
107
+ markers=True,
108
+ custom_data=["task", "score", "model"],
109
+ title=title,
110
+ )
111
+
112
+ # Update hovertemplate for better hover interaction experience
113
+ fig.update_traces(
114
+ hovertemplate="<br>".join(
115
+ [
116
+ "Model Name: %{customdata[2]}",
117
+ "Metric Name: %{customdata[0]}",
118
+ "Date: %{x}",
119
+ "Metric Value: %{y}",
120
+ ]
121
+ )
122
+ )
123
+
124
+ # Update the range of the y-axis
125
+ fig.update_layout(yaxis_range=[0, 100])
126
+
127
+ # Create a dictionary to hold the color mapping for each metric
128
+ metric_color_mapping = {}
129
+
130
+ # Map each metric name to its color in the figure
131
+ for trace in fig.data:
132
+ metric_color_mapping[trace.name] = trace.line.color
133
+
134
+ # Iterate over filtered human baselines and add horizontal lines to the figure
135
+ for metric, value in filtered_human_baselines.items():
136
+ color = metric_color_mapping.get(metric, "blue") # Retrieve color from mapping; default to blue if not found
137
+ location = "top left" if metric == "HellaSwag" else "bottom left" # Set annotation position
138
+ # Add horizontal line with matched color and positioned annotation
139
+ fig.add_hline(
140
+ y=value,
141
+ line_dash="dot",
142
+ annotation_text=f"{metric} human baseline",
143
+ annotation_position=location,
144
+ annotation_font_size=10,
145
+ annotation_font_color=color,
146
+ line_color=color,
147
+ )
148
+
149
+ return fig
150
+
151
+
152
+ # Example Usage:
153
+ # human_baselines dictionary is defined.
154
+ # chart = create_metric_plot_obj(scores_df, ["ARC", "HellaSwag", "MMLU", "TruthfulQA"], human_baselines, "Graph Title")