BenchmarkBot commited on
Commit
5490c7c
β€’
1 Parent(s): a894537

sort by tradeoff but don't show it

Browse files
Files changed (1) hide show
  1. app.py +16 -7
app.py CHANGED
@@ -59,7 +59,7 @@ ALL_COLUMNS_DATATYPES = [
59
  "number",
60
  "number",
61
  ]
62
- SORTING_COLUMN = ["Score (%) ⬆️"]
63
 
64
  llm_perf_dataset_repo = load_dataset_repo(LLM_PERF_DATASET_REPO, OPTIMUM_TOKEN)
65
 
@@ -73,10 +73,10 @@ def get_benchmark_df(benchmark="1xA100-80GB"):
73
  scores_df = pd.read_csv(
74
  f"./llm-perf-dataset/reports/Grouped-Open-LLM-Leaderboard.csv"
75
  )
76
- bench_df = bench_df.merge(scores_df, left_on="model", right_on="best_scored_model")
77
 
78
  # add optimizations
79
- bench_df["optimizations"] = bench_df[
80
  ["backend.bettertransformer", "backend.load_in_8bit", "backend.load_in_4bit"]
81
  ].apply(
82
  lambda x: ", ".join(
@@ -94,16 +94,23 @@ def get_benchmark_df(benchmark="1xA100-80GB"):
94
  axis=1,
95
  )
96
 
97
- return bench_df
 
 
 
 
 
 
 
98
 
99
 
100
  def get_benchmark_table(bench_df):
 
 
101
  # filter
102
  bench_df = bench_df[list(ALL_COLUMNS_MAPPING.keys())]
103
  # rename
104
  bench_df.rename(columns=ALL_COLUMNS_MAPPING, inplace=True)
105
- # sort
106
- bench_df.sort_values(by=SORTING_COLUMN, ascending=True, inplace=True)
107
  # transform
108
  bench_df["Model Type πŸ€—"] = bench_df["Model Type πŸ€—"].apply(process_model_type)
109
  bench_df["Weight Class πŸ‹οΈ"] = bench_df["Weight Class πŸ‹οΈ"].apply(
@@ -223,7 +230,9 @@ with demo:
223
 
224
  # leaderboard tabs
225
  with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:
226
- with gr.TabItem("πŸ–₯️ A100-80GB Leaderboard Table πŸ…", id=0):
 
 
227
  gr.HTML(A100_TEXT)
228
 
229
  # Original leaderboard table
 
59
  "number",
60
  "number",
61
  ]
62
+ SORTING_COLUMN = ["tradeoff"]
63
 
64
  llm_perf_dataset_repo = load_dataset_repo(LLM_PERF_DATASET_REPO, OPTIMUM_TOKEN)
65
 
 
73
  scores_df = pd.read_csv(
74
  f"./llm-perf-dataset/reports/Grouped-Open-LLM-Leaderboard.csv"
75
  )
76
+ merged_df = bench_df.merge(scores_df, left_on="model", right_on="best_scored_model")
77
 
78
  # add optimizations
79
+ merged_df["optimizations"] = merged_df[
80
  ["backend.bettertransformer", "backend.load_in_8bit", "backend.load_in_4bit"]
81
  ].apply(
82
  lambda x: ", ".join(
 
94
  axis=1,
95
  )
96
 
97
+ # create composite score
98
+ score_distance = 100 - merged_df["best_score"]
99
+ # normalize latency between 0 and 100
100
+ latency_distance = merged_df["generate.latency(s)"]
101
+ merged_df["tradeoff"] = (score_distance**2 + latency_distance**2) ** 0.5
102
+ merged_df["tradeoff"] = merged_df["tradeoff"].round(2)
103
+
104
+ return merged_df
105
 
106
 
107
  def get_benchmark_table(bench_df):
108
+ # sort
109
+ bench_df.sort_values(by=SORTING_COLUMN, ascending=True, inplace=True)
110
  # filter
111
  bench_df = bench_df[list(ALL_COLUMNS_MAPPING.keys())]
112
  # rename
113
  bench_df.rename(columns=ALL_COLUMNS_MAPPING, inplace=True)
 
 
114
  # transform
115
  bench_df["Model Type πŸ€—"] = bench_df["Model Type πŸ€—"].apply(process_model_type)
116
  bench_df["Weight Class πŸ‹οΈ"] = bench_df["Weight Class πŸ‹οΈ"].apply(
 
230
 
231
  # leaderboard tabs
232
  with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:
233
+ with gr.TabItem(
234
+ "πŸ–₯️ A100-80GB Leaderboar Table οΏ½πŸ…πŸ† πŸ…οΏ½eπŸ…eπŸ… πŸ†πŸ…οΏ½πŸ…πŸ† πŸ…οΏ½eπŸ…eπŸ… πŸ†", id=0
235
+ ):
236
  gr.HTML(A100_TEXT)
237
 
238
  # Original leaderboard table