BenchmarkBot commited on
Commit
483e3a1
โ€ข
1 Parent(s): e89d633

added example config

Browse files
Files changed (2) hide show
  1. app.py +2 -0
  2. src/assets/text_content.py +46 -1
app.py CHANGED
@@ -9,6 +9,7 @@ from src.assets.text_content import (
9
  TITLE,
10
  INTRODUCTION_TEXT,
11
  ABOUT_TEXT,
 
12
  CITATION_BUTTON_LABEL,
13
  CITATION_BUTTON_TEXT,
14
  )
@@ -335,6 +336,7 @@ with demo:
335
 
336
  with gr.TabItem("About ๐Ÿ“–", id=3):
337
  gr.HTML(ABOUT_TEXT, elem_classes="descriptive-text")
 
338
 
339
  demo.load(
340
  change_tab,
 
9
  TITLE,
10
  INTRODUCTION_TEXT,
11
  ABOUT_TEXT,
12
+ EXAMPLE_CONFIG_TEXT,
13
  CITATION_BUTTON_LABEL,
14
  CITATION_BUTTON_TEXT,
15
  )
 
336
 
337
  with gr.TabItem("About ๐Ÿ“–", id=3):
338
  gr.HTML(ABOUT_TEXT, elem_classes="descriptive-text")
339
+ gr.Markdown(EXAMPLE_CONFIG_TEXT, elem_classes="descriptive-text")
340
 
341
  demo.load(
342
  change_tab,
src/assets/text_content.py CHANGED
@@ -15,10 +15,55 @@ ABOUT_TEXT = """<h3>About the ๐Ÿค— Open LLM-Perf Leaderboard ๐Ÿ‹๏ธ</h3>
15
  <li>Peak memory is measured in MB during the first forward pass of the LLM (no warmup).</li>
16
  <li>Each pair of (Model Type, Weight Class) is represented by the best scored model. This LLM is the one used for all the hardware/backend/optimization experiments.</li>
17
  <li>Score is the average evaluation score obtained from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">๐Ÿค— Open LLM Leaderboard</a>.</li>
18
- <li>Ranking is based on the euclidean distance from the "Perfect LLM" (i.e. 0 latency and 100% accuracy).</li>
19
  </ul>
20
  """
21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results."
23
  CITATION_BUTTON_TEXT = r"""@misc{open-llm-perf-leaderboard,
24
  author = {Ilyas Moutawwakil, Rรฉgis Pierrard},
 
15
  <li>Peak memory is measured in MB during the first forward pass of the LLM (no warmup).</li>
16
  <li>Each pair of (Model Type, Weight Class) is represented by the best scored model. This LLM is the one used for all the hardware/backend/optimization experiments.</li>
17
  <li>Score is the average evaluation score obtained from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">๐Ÿค— Open LLM Leaderboard</a>.</li>
18
+ <li>Ranking is based on a composite metric which is the euclidean distance from the "Perfect LLM" (i.e. 0 latency and 100% accuracy).</li>
19
  </ul>
20
  """
21
 
22
+ EXAMPLE_CONFIG_TEXT = """
23
+ Here's an example of the configuration file used to benchmark the models with Optimum-Benchmark:
24
+ ```yaml
25
+ defaults:
26
+ - backend: pytorch # default backend
27
+ - benchmark: inference # default benchmark
28
+ - experiment # inheriting from experiment config
29
+ - _self_ # for hydra 1.1 compatibility
30
+ - override hydra/job_logging: colorlog # colorful logging
31
+ - override hydra/hydra_logging: colorlog # colorful logging
32
+
33
+ hydra:
34
+ run:
35
+ dir: llm-experiments/{experiment_name}
36
+ job:
37
+ chdir: true
38
+
39
+ experiment_name: {experiment_name}
40
+
41
+ model: {model}
42
+ hub_kwargs:
43
+ revision: {revision}
44
+ trust_remote_code: {trust_remote_code}
45
+
46
+ device: cuda
47
+
48
+ backend:
49
+ no_weights: true
50
+ delete_cache: true
51
+ torch_dtype: float16
52
+ load_in_4bit: true
53
+ bettertransformer: true
54
+
55
+ benchmark:
56
+ memory: true
57
+
58
+ input_shapes:
59
+ batch_size: 1
60
+ sequence_length: 1
61
+
62
+ new_tokens: 1000
63
+ ```
64
+ """
65
+
66
+
67
  CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results."
68
  CITATION_BUTTON_TEXT = r"""@misc{open-llm-perf-leaderboard,
69
  author = {Ilyas Moutawwakil, Rรฉgis Pierrard},