BenchmarkBot commited on
Commit
699b4cd
β€’
1 Parent(s): 06b3632

add quantizations

Browse files
Files changed (2) hide show
  1. app.py +24 -13
  2. src/assets/css_html_js.py +0 -8
app.py CHANGED
@@ -4,7 +4,7 @@ import pandas as pd
4
  import plotly.express as px
5
  from apscheduler.schedulers.background import BackgroundScheduler
6
 
7
- from src.assets.css_html_js import custom_css, custom_js
8
  from src.assets.text_content import (
9
  TITLE,
10
  INTRODUCTION_TEXT,
@@ -14,7 +14,6 @@ from src.assets.text_content import (
14
  CITATION_BUTTON_TEXT,
15
  )
16
  from src.utils import (
17
- change_tab,
18
  restart_space,
19
  load_dataset_repo,
20
  process_model_name,
@@ -114,9 +113,8 @@ def get_benchmark_df(benchmark="Succeeded-1xA100-80GB"):
114
 
115
 
116
  def get_benchmark_table(bench_df):
117
- # add * to quantized models score
118
  copy_df = bench_df.copy()
119
- # add * to quantized models score since we can't garantee the score is the same
120
  copy_df["best_score"] = copy_df.apply(
121
  lambda x: f"{x['best_score']}**"
122
  if x["backend.quantization_strategy"] in ["bnb", "gptq"]
@@ -179,6 +177,7 @@ def filter_query(
179
  backends,
180
  datatypes,
181
  optimizations,
 
182
  score,
183
  memory,
184
  benchmark="Succeeded-1xA100-80GB",
@@ -199,6 +198,17 @@ def filter_query(
199
  if len(optimizations) > 0
200
  else True
201
  )
 
 
 
 
 
 
 
 
 
 
 
202
  & (raw_df["best_score"] >= score)
203
  & (raw_df["forward.peak_memory(MB)"] <= memory)
204
  ]
@@ -277,7 +287,6 @@ with demo:
277
  value=80 * 1024,
278
  elem_id="memory-slider",
279
  )
280
-
281
  with gr.Row():
282
  with gr.Column(scale=1):
283
  backend_checkboxes = gr.CheckboxGroup(
@@ -295,7 +304,7 @@ with demo:
295
  info="β˜‘οΈ Select the load dtypes",
296
  elem_id="dtype-checkboxes",
297
  )
298
- with gr.Column(scale=2):
299
  optimizations_checkboxes = gr.CheckboxGroup(
300
  label="Optimizations πŸ› οΈ",
301
  choices=["None", "BetterTransformer"],
@@ -303,7 +312,14 @@ with demo:
303
  info="β˜‘οΈ Select the optimizations",
304
  elem_id="optimizations-checkboxes",
305
  )
306
-
 
 
 
 
 
 
 
307
  with gr.Row():
308
  filter_button = gr.Button(
309
  value="Filter πŸš€",
@@ -314,12 +330,6 @@ with demo:
314
  gr.HTML(ABOUT_TEXT, elem_classes="descriptive-text")
315
  gr.Markdown(EXAMPLE_CONFIG_TEXT, elem_classes="descriptive-text")
316
 
317
- demo.load(
318
- change_tab,
319
- A100_tabs,
320
- _js=custom_js,
321
- )
322
-
323
  filter_button.click(
324
  filter_query,
325
  [
@@ -327,6 +337,7 @@ with demo:
327
  backend_checkboxes,
328
  datatype_checkboxes,
329
  optimizations_checkboxes,
 
330
  score_slider,
331
  memory_slider,
332
  ],
 
4
  import plotly.express as px
5
  from apscheduler.schedulers.background import BackgroundScheduler
6
 
7
+ from src.assets.css_html_js import custom_css
8
  from src.assets.text_content import (
9
  TITLE,
10
  INTRODUCTION_TEXT,
 
14
  CITATION_BUTTON_TEXT,
15
  )
16
  from src.utils import (
 
17
  restart_space,
18
  load_dataset_repo,
19
  process_model_name,
 
113
 
114
 
115
  def get_benchmark_table(bench_df):
 
116
  copy_df = bench_df.copy()
117
+ # adding ** to quantized models score since we can't garantee the score is the same
118
  copy_df["best_score"] = copy_df.apply(
119
  lambda x: f"{x['best_score']}**"
120
  if x["backend.quantization_strategy"] in ["bnb", "gptq"]
 
177
  backends,
178
  datatypes,
179
  optimizations,
180
+ quantization_scheme,
181
  score,
182
  memory,
183
  benchmark="Succeeded-1xA100-80GB",
 
198
  if len(optimizations) > 0
199
  else True
200
  )
201
+ & (
202
+ pd.concat(
203
+ [
204
+ raw_df["quantization"] == quantization
205
+ for quantization in quantization_scheme
206
+ ],
207
+ axis=1,
208
+ ).any(axis="columns")
209
+ if len(quantization_scheme) > 0
210
+ else True
211
+ )
212
  & (raw_df["best_score"] >= score)
213
  & (raw_df["forward.peak_memory(MB)"] <= memory)
214
  ]
 
287
  value=80 * 1024,
288
  elem_id="memory-slider",
289
  )
 
290
  with gr.Row():
291
  with gr.Column(scale=1):
292
  backend_checkboxes = gr.CheckboxGroup(
 
304
  info="β˜‘οΈ Select the load dtypes",
305
  elem_id="dtype-checkboxes",
306
  )
307
+ with gr.Column(scale=1):
308
  optimizations_checkboxes = gr.CheckboxGroup(
309
  label="Optimizations πŸ› οΈ",
310
  choices=["None", "BetterTransformer"],
 
312
  info="β˜‘οΈ Select the optimizations",
313
  elem_id="optimizations-checkboxes",
314
  )
315
+ with gr.Column(scale=1):
316
+ quantization_checkboxes = gr.CheckboxGroup(
317
+ label="Quantization πŸ—œοΈ",
318
+ choices=["None", "BnB.4bit", "GPTQ.4bit"],
319
+ value=["None", "BnB.4bit", "GPTQ.4bit"],
320
+ info="β˜‘οΈ Select the quantization schemes",
321
+ elem_id="quantization-checkboxes",
322
+ )
323
  with gr.Row():
324
  filter_button = gr.Button(
325
  value="Filter πŸš€",
 
330
  gr.HTML(ABOUT_TEXT, elem_classes="descriptive-text")
331
  gr.Markdown(EXAMPLE_CONFIG_TEXT, elem_classes="descriptive-text")
332
 
 
 
 
 
 
 
333
  filter_button.click(
334
  filter_query,
335
  [
 
337
  backend_checkboxes,
338
  datatype_checkboxes,
339
  optimizations_checkboxes,
340
+ quantization_checkboxes,
341
  score_slider,
342
  memory_slider,
343
  ],
src/assets/css_html_js.py CHANGED
@@ -23,11 +23,3 @@ custom_css = """
23
  font-size: 20px;
24
  }
25
  """
26
-
27
- custom_js = """
28
- function(url_params) {
29
- const params = new URLSearchParams(window.location.search);
30
- url_params = Object.fromEntries(params);
31
- return url_params;
32
- }
33
- """
 
23
  font-size: 20px;
24
  }
25
  """