Spaces:
Running
Running
Commit
β’
75b2595
1
Parent(s):
18468e8
fix int4
Browse files
app.py
CHANGED
@@ -68,14 +68,10 @@ def get_benchmark_df(benchmark="1xA100-80GB"):
|
|
68 |
bench_df["optimizations"] = bench_df[
|
69 |
["backend.bettertransformer", "backend.load_in_8bit", "backend.load_in_4bit"]
|
70 |
].apply(
|
71 |
-
lambda x: "
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
["BetterTransformer", "LLM.int8", "NF4"], x.values
|
76 |
-
)
|
77 |
-
]
|
78 |
-
),
|
79 |
)
|
80 |
|
81 |
return bench_df
|
@@ -99,7 +95,7 @@ def get_benchmark_table(bench_df):
|
|
99 |
|
100 |
def get_benchmark_plot(bench_df):
|
101 |
# untill falcon gets fixed / natively supported
|
102 |
-
bench_df = bench_df[bench_df["generate.latency(s)"] <
|
103 |
|
104 |
fig = px.scatter(
|
105 |
bench_df,
|
@@ -117,7 +113,6 @@ def get_benchmark_plot(bench_df):
|
|
117 |
"generate.throughput(tokens/s)",
|
118 |
],
|
119 |
symbol_sequence=["triangle-up", "circle"],
|
120 |
-
# as many distinct colors as there are model_type,backend.name couples
|
121 |
color_discrete_sequence=px.colors.qualitative.Light24,
|
122 |
)
|
123 |
|
@@ -254,8 +249,8 @@ with demo:
|
|
254 |
with gr.Column(scale=2):
|
255 |
optimizations_checkboxes = gr.CheckboxGroup(
|
256 |
label="Optimizations π οΈ",
|
257 |
-
choices=["None", "BetterTransformer", "LLM.int8", "
|
258 |
-
value=["None", "BetterTransformer", "LLM.int8", "
|
259 |
info="βοΈ Select the optimizations",
|
260 |
elem_id="optimizations-checkboxes",
|
261 |
)
|
|
|
68 |
bench_df["optimizations"] = bench_df[
|
69 |
["backend.bettertransformer", "backend.load_in_8bit", "backend.load_in_4bit"]
|
70 |
].apply(
|
71 |
+
lambda x: "BetterTransformer"
|
72 |
+
if x[0] == True
|
73 |
+
else ("LLM.int8" if x[1] == True else ("LLM.fp4" if x[2] == True else "None")),
|
74 |
+
axis=1,
|
|
|
|
|
|
|
|
|
75 |
)
|
76 |
|
77 |
return bench_df
|
|
|
95 |
|
96 |
def get_benchmark_plot(bench_df):
|
97 |
# untill falcon gets fixed / natively supported
|
98 |
+
bench_df = bench_df[bench_df["generate.latency(s)"] < 100]
|
99 |
|
100 |
fig = px.scatter(
|
101 |
bench_df,
|
|
|
113 |
"generate.throughput(tokens/s)",
|
114 |
],
|
115 |
symbol_sequence=["triangle-up", "circle"],
|
|
|
116 |
color_discrete_sequence=px.colors.qualitative.Light24,
|
117 |
)
|
118 |
|
|
|
249 |
with gr.Column(scale=2):
|
250 |
optimizations_checkboxes = gr.CheckboxGroup(
|
251 |
label="Optimizations π οΈ",
|
252 |
+
choices=["None", "BetterTransformer", "LLM.int8", "LLM.fp4"],
|
253 |
+
value=["None", "BetterTransformer", "LLM.int8", "LLM.fp4"],
|
254 |
info="βοΈ Select the optimizations",
|
255 |
elem_id="optimizations-checkboxes",
|
256 |
)
|