BenchmarkBot commited on
Commit
dcfabfb
1 Parent(s): d912876

updated dataset

Browse files
Files changed (1) hide show
  1. app.py +13 -17
app.py CHANGED
@@ -12,14 +12,14 @@ LLM_PERF_LEADERBOARD_REPO = "optimum/llm-perf-leaderboard"
12
  LLM_PERF_DATASET_REPO = "optimum/llm-perf-dataset"
13
  OPTIMUM_TOKEN = os.environ.get("OPTIMUM_TOKEN")
14
 
15
- OLD_COLUMNS = ["model", "backend.name", "backend.torch_dtype",
16
- "generate.latency(s)", "generate.throughput(tokens/s)"]
17
-
18
- NEW_COLUMNS = ["Model", "Backend 🏭", "Load Datatype",
19
- "Latency (s) ⬇️", "Throughput (tokens/s) ⬆️"]
20
-
 
21
  COLUMNS_DATATYPES = ["markdown", "str", "str", "number", "number"]
22
-
23
  SORTING_COLUMN = ["Throughput (tokens/s) ⬆️"]
24
 
25
 
@@ -31,20 +31,15 @@ def get_benchmark_df(benchmark):
31
  llm_perf_dataset_repo.git_pull()
32
 
33
  # load
34
- df = pd.read_csv(
35
- f"./llm-perf-dataset/reports/{benchmark}/inference_report.csv")
36
-
37
  # preprocess
38
  df["model"] = df["model"].apply(make_clickable_model)
39
-
40
  # filter
41
- df = df[OLD_COLUMNS]
42
-
43
  # rename
44
  df.rename(columns={
45
- df_col: rename_col for df_col, rename_col in zip(OLD_COLUMNS, NEW_COLUMNS)
46
  }, inplace=True)
47
-
48
  # sort
49
  df.sort_values(by=SORTING_COLUMN, ascending=False, inplace=True)
50
 
@@ -72,7 +67,7 @@ with demo:
72
  leaderboard_table_lite = gr.components.Dataframe(
73
  value=single_A100_df,
74
  datatype=COLUMNS_DATATYPES,
75
- headers=NEW_COLUMNS,
76
  elem_id="1xA100-table",
77
  )
78
 
@@ -87,7 +82,7 @@ with demo:
87
  leaderboard_table_full = gr.components.Dataframe(
88
  value=multi_A100_df,
89
  datatype=COLUMNS_DATATYPES,
90
- headers=NEW_COLUMNS,
91
  elem_id="4xA100-table",
92
  )
93
 
@@ -100,6 +95,7 @@ with demo:
100
  elem_id="citation-button",
101
  ).style(show_copy_button=True)
102
 
 
103
  # Restart space every hour
104
  scheduler = BackgroundScheduler()
105
  scheduler.add_job(restart_space, "interval", seconds=3600,
 
12
  LLM_PERF_DATASET_REPO = "optimum/llm-perf-dataset"
13
  OPTIMUM_TOKEN = os.environ.get("OPTIMUM_TOKEN")
14
 
15
+ COLUMNS_MAPPING = {
16
+ "model": "Model 🤗",
17
+ "backend.name": "Backend 🏭",
18
+ "backend.torch_dtype": "Load Datatype 📥",
19
+ "generate.latency(s)": "Latency (s) ⬇️",
20
+ "generate.throughput(tokens/s)": "Throughput (tokens/s) ⬆️",
21
+ }
22
  COLUMNS_DATATYPES = ["markdown", "str", "str", "number", "number"]
 
23
  SORTING_COLUMN = ["Throughput (tokens/s) ⬆️"]
24
 
25
 
 
31
  llm_perf_dataset_repo.git_pull()
32
 
33
  # load
34
+ df = pd.read_csv(f"llm-perf-dataset/reports/{benchmark}/inference_report.csv")
 
 
35
  # preprocess
36
  df["model"] = df["model"].apply(make_clickable_model)
 
37
  # filter
38
+ df = df[COLUMNS_MAPPING.keys()]
 
39
  # rename
40
  df.rename(columns={
41
+ df_col: rename_col for df_col, rename_col in COLUMNS_MAPPING.items()
42
  }, inplace=True)
 
43
  # sort
44
  df.sort_values(by=SORTING_COLUMN, ascending=False, inplace=True)
45
 
 
67
  leaderboard_table_lite = gr.components.Dataframe(
68
  value=single_A100_df,
69
  datatype=COLUMNS_DATATYPES,
70
+ headers=COLUMNS_MAPPING.values(),
71
  elem_id="1xA100-table",
72
  )
73
 
 
82
  leaderboard_table_full = gr.components.Dataframe(
83
  value=multi_A100_df,
84
  datatype=COLUMNS_DATATYPES,
85
+ headers=COLUMNS_MAPPING.values(),
86
  elem_id="4xA100-table",
87
  )
88
 
 
95
  elem_id="citation-button",
96
  ).style(show_copy_button=True)
97
 
98
+
99
  # Restart space every hour
100
  scheduler = BackgroundScheduler()
101
  scheduler.add_job(restart_space, "interval", seconds=3600,