Hugging Face's logo

kreas
/
FA_benchmarks

Model card Files Community
kreas commited on
Commit
db6805d
•
1 Parent(s): 947fe08

Upload /test-mistral-7b-v0.1-awq/AWQ_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. test-mistral-7b-v0.1-awq/AWQ_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +351 -0
test-mistral-7b-v0.1-awq/AWQ_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json ADDED
@@ -0,0 +1,351 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "prefill": {
3
+ "memory": {
4
+ "unit": "MB",
5
+ "max_ram": 2027.565056,
6
+ "max_vram": 5981.077504,
7
+ "max_reserved": 5496.635392,
8
+ "max_allocated": 5273.977856
9
+ },
10
+ "latency": {
11
+ "unit": "s",
12
+ "mean": 0.3072904746315696,
13
+ "stdev": 0.0009002358294454896,
14
+ "values": [
15
+ 0.31204364013671876,
16
+ 0.3088294372558594,
17
+ 0.3071784973144531,
18
+ 0.30689688110351565,
19
+ 0.30723687744140626,
20
+ 0.3069060974121094,
21
+ 0.3071774597167969,
22
+ 0.30688870239257815,
23
+ 0.30710784912109373,
24
+ 0.30693499755859377,
25
+ 0.3071641540527344,
26
+ 0.306977783203125,
27
+ 0.307135498046875,
28
+ 0.30694708251953123,
29
+ 0.30717953491210936,
30
+ 0.3069859924316406,
31
+ 0.30723480224609373,
32
+ 0.30689483642578125,
33
+ 0.3072337951660156,
34
+ 0.3071703186035156,
35
+ 0.30717953491210936,
36
+ 0.3069378662109375,
37
+ 0.307156982421875,
38
+ 0.30701873779296873,
39
+ 0.307209228515625,
40
+ 0.3071150207519531,
41
+ 0.30723583984375,
42
+ 0.30697882080078126,
43
+ 0.30720315551757815,
44
+ 0.3069204406738281,
45
+ 0.30727883911132814,
46
+ 0.3070136413574219,
47
+ 0.30721331787109374
48
+ ]
49
+ },
50
+ "throughput": {
51
+ "unit": "tokens/s",
52
+ "value": 833.0879774484872
53
+ },
54
+ "energy": null,
55
+ "efficiency": null
56
+ },
57
+ "decode": {
58
+ "memory": {
59
+ "unit": "MB",
60
+ "max_ram": 2027.565056,
61
+ "max_vram": 6234.832896,
62
+ "max_reserved": 5750.390784,
63
+ "max_allocated": 5425.04704
64
+ },
65
+ "latency": {
66
+ "unit": "s",
67
+ "mean": 76.5955501708985,
68
+ "stdev": 0,
69
+ "values": [
70
+ 76.5955501708985
71
+ ]
72
+ },
73
+ "throughput": {
74
+ "unit": "tokens/s",
75
+ "value": 3.3291751208921267
76
+ },
77
+ "energy": null,
78
+ "efficiency": null
79
+ },
80
+ "per_token": {
81
+ "memory": null,
82
+ "latency": {
83
+ "unit": "s",
84
+ "mean": 0.30037470655254317,
85
+ "stdev": 0.0004994743161792463,
86
+ "values": [
87
+ 0.2996787109375,
88
+ 0.3002675170898437,
89
+ 0.3002941589355469,
90
+ 0.3001466979980469,
91
+ 0.30035968017578124,
92
+ 0.30015692138671873,
93
+ 0.300303466796875,
94
+ 0.30024386596679686,
95
+ 0.30030950927734373,
96
+ 0.3004876708984375,
97
+ 0.30025027465820314,
98
+ 0.30013116455078126,
99
+ 0.3003504638671875,
100
+ 0.3001743469238281,
101
+ 0.30021221923828123,
102
+ 0.30009036254882815,
103
+ 0.30017535400390627,
104
+ 0.3002654724121094,
105
+ 0.30025112915039065,
106
+ 0.30015078735351564,
107
+ 0.3001958312988281,
108
+ 0.3003187255859375,
109
+ 0.30023373413085935,
110
+ 0.30013131713867186,
111
+ 0.30027572631835936,
112
+ 0.30018560791015625,
113
+ 0.3002101745605469,
114
+ 0.30023678588867186,
115
+ 0.300237060546875,
116
+ 0.30014437866210936,
117
+ 0.30024295043945315,
118
+ 0.30024395751953126,
119
+ 0.300142578125,
120
+ 0.30021221923828123,
121
+ 0.30026239013671874,
122
+ 0.3001978759765625,
123
+ 0.30039141845703127,
124
+ 0.30019277954101564,
125
+ 0.30021630859375,
126
+ 0.30015896606445314,
127
+ 0.3002808227539063,
128
+ 0.30024191284179685,
129
+ 0.30030337524414064,
130
+ 0.30030337524414064,
131
+ 0.30030438232421874,
132
+ 0.30031768798828123,
133
+ 0.3002296447753906,
134
+ 0.30030337524414064,
135
+ 0.30023577880859376,
136
+ 0.30034738159179686,
137
+ 0.30030950927734373,
138
+ 0.3002828674316406,
139
+ 0.3002767333984375,
140
+ 0.30078155517578126,
141
+ 0.3002941589355469,
142
+ 0.30017843627929686,
143
+ 0.3006443481445312,
144
+ 0.30029107666015625,
145
+ 0.3003166809082031,
146
+ 0.30034124755859376,
147
+ 0.30034124755859376,
148
+ 0.3003309936523437,
149
+ 0.3002501220703125,
150
+ 0.3002398681640625,
151
+ 0.303035400390625,
152
+ 0.30368154907226563,
153
+ 0.30227557373046876,
154
+ 0.30051431274414064,
155
+ 0.3003975830078125,
156
+ 0.3003289489746094,
157
+ 0.300179443359375,
158
+ 0.30040576171875,
159
+ 0.30029107666015625,
160
+ 0.30027877807617187,
161
+ 0.30025421142578124,
162
+ 0.30032794189453127,
163
+ 0.3002245178222656,
164
+ 0.3021414489746094,
165
+ 0.3011962890625,
166
+ 0.3002828674316406,
167
+ 0.30034228515625,
168
+ 0.300400634765625,
169
+ 0.3002593383789062,
170
+ 0.30037298583984373,
171
+ 0.3002388610839844,
172
+ 0.30037503051757813,
173
+ 0.3013877868652344,
174
+ 0.30037298583984373,
175
+ 0.30024090576171875,
176
+ 0.300379150390625,
177
+ 0.3002736511230469,
178
+ 0.30043649291992186,
179
+ 0.3001692199707031,
180
+ 0.30030029296875,
181
+ 0.3002767333984375,
182
+ 0.3002552185058594,
183
+ 0.3001978759765625,
184
+ 0.30015896606445314,
185
+ 0.30031256103515624,
186
+ 0.3002132568359375,
187
+ 0.30029852294921877,
188
+ 0.3003092041015625,
189
+ 0.30029925537109375,
190
+ 0.30020913696289064,
191
+ 0.3001978759765625,
192
+ 0.30026239013671874,
193
+ 0.3001570739746094,
194
+ 0.3003380126953125,
195
+ 0.3060234375,
196
+ 0.3004067687988281,
197
+ 0.30030438232421874,
198
+ 0.30035354614257814,
199
+ 0.30030029296875,
200
+ 0.300432373046875,
201
+ 0.30024295043945315,
202
+ 0.30030950927734373,
203
+ 0.30017230224609376,
204
+ 0.30034124755859376,
205
+ 0.3002081298828125,
206
+ 0.30031256103515624,
207
+ 0.300147705078125,
208
+ 0.30026443481445314,
209
+ 0.30023373413085935,
210
+ 0.3004344177246094,
211
+ 0.3002668151855469,
212
+ 0.3005252380371094,
213
+ 0.3001763916015625,
214
+ 0.30029824829101565,
215
+ 0.30042520141601564,
216
+ 0.3004354553222656,
217
+ 0.30034738159179686,
218
+ 0.30045901489257815,
219
+ 0.3003084716796875,
220
+ 0.3003709411621094,
221
+ 0.30020095825195314,
222
+ 0.3003105163574219,
223
+ 0.3003904113769531,
224
+ 0.30039450073242185,
225
+ 0.3002603454589844,
226
+ 0.30034228515625,
227
+ 0.3002296447753906,
228
+ 0.3003484191894531,
229
+ 0.30020709228515624,
230
+ 0.3003146362304687,
231
+ 0.3002112121582031,
232
+ 0.30042315673828124,
233
+ 0.3001548767089844,
234
+ 0.300305419921875,
235
+ 0.300358642578125,
236
+ 0.30032177734375,
237
+ 0.30038119506835936,
238
+ 0.30042520141601564,
239
+ 0.30043649291992186,
240
+ 0.30041702270507814,
241
+ 0.3003074645996094,
242
+ 0.3003699340820313,
243
+ 0.30043136596679687,
244
+ 0.3002716064453125,
245
+ 0.3003934631347656,
246
+ 0.3002296447753906,
247
+ 0.3002818603515625,
248
+ 0.3001845703125,
249
+ 0.3002193908691406,
250
+ 0.3004293212890625,
251
+ 0.30044467163085936,
252
+ 0.30032794189453127,
253
+ 0.30038836669921876,
254
+ 0.3003555908203125,
255
+ 0.30035150146484374,
256
+ 0.30023065185546877,
257
+ 0.3004190673828125,
258
+ 0.3002173461914063,
259
+ 0.3007559814453125,
260
+ 0.3003074645996094,
261
+ 0.3003504638671875,
262
+ 0.3004549255371094,
263
+ 0.3007508544921875,
264
+ 0.30035250854492185,
265
+ 0.3004172058105469,
266
+ 0.30032672119140624,
267
+ 0.3004344177246094,
268
+ 0.300337158203125,
269
+ 0.3004190673828125,
270
+ 0.3003443298339844,
271
+ 0.3003996276855469,
272
+ 0.3003862915039063,
273
+ 0.30038836669921876,
274
+ 0.300326904296875,
275
+ 0.3003719787597656,
276
+ 0.30025830078125,
277
+ 0.30038323974609377,
278
+ 0.30027877807617187,
279
+ 0.30038323974609377,
280
+ 0.3003904113769531,
281
+ 0.3001773986816406,
282
+ 0.3002326965332031,
283
+ 0.3001540222167969,
284
+ 0.3003287963867188,
285
+ 0.30025830078125,
286
+ 0.30023577880859376,
287
+ 0.30079898071289063,
288
+ 0.3002654724121094,
289
+ 0.30028594970703126,
290
+ 0.3003873291015625,
291
+ 0.3002767333984375,
292
+ 0.3005388793945313,
293
+ 0.3002094421386719,
294
+ 0.30033685302734375,
295
+ 0.3002941589355469,
296
+ 0.3002501220703125,
297
+ 0.3003013000488281,
298
+ 0.30030438232421874,
299
+ 0.30018765258789065,
300
+ 0.30033407592773437,
301
+ 0.3002101745605469,
302
+ 0.300295166015625,
303
+ 0.30029107666015625,
304
+ 0.30040985107421875,
305
+ 0.30038836669921876,
306
+ 0.30032177734375,
307
+ 0.30028390502929686,
308
+ 0.3003187255859375,
309
+ 0.3002173461914063,
310
+ 0.3003084716796875,
311
+ 0.30030438232421874,
312
+ 0.30043136596679687,
313
+ 0.30018865966796876,
314
+ 0.30030029296875,
315
+ 0.30015896606445314,
316
+ 0.3002736511230469,
317
+ 0.30024090576171875,
318
+ 0.30035354614257814,
319
+ 0.3011307373046875,
320
+ 0.30041497802734374,
321
+ 0.3003381652832031,
322
+ 0.30034637451171875,
323
+ 0.3002296447753906,
324
+ 0.3003166809082031,
325
+ 0.30021426391601563,
326
+ 0.3003351135253906,
327
+ 0.3001610107421875,
328
+ 0.30036685180664063,
329
+ 0.30023373413085935,
330
+ 0.30026138305664063,
331
+ 0.300337158203125,
332
+ 0.30042315673828124,
333
+ 0.30027880859375,
334
+ 0.30027566528320315,
335
+ 0.3001128845214844,
336
+ 0.30024090576171875,
337
+ 0.3002746887207031,
338
+ 0.30023065185546877,
339
+ 0.30012109375,
340
+ 0.3007569885253906,
341
+ 0.30015896606445314
342
+ ]
343
+ },
344
+ "throughput": {
345
+ "unit": "tokens/s",
346
+ "value": 3.3291751208921267
347
+ },
348
+ "energy": null,
349
+ "efficiency": null
350
+ }
351
+ }