kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Mar 14

Commit

7dd07f7

•

1 Parent(s): 8d0bd14

Upload /Mistral-7B-v0.1-AWQ/AWQ_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Mistral-7B-v0.1-AWQ/AWQ_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +306 -795

Mistral-7B-v0.1-AWQ/AWQ_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,287 +2,54 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 991.27296,
-            "max_vram": 2585.788416,
-            "max_reserved": 2101.346304,
-            "max_allocated": 2014.912
         },
         "latency": {
             "unit": "s",
-            "mean": 0.03763389647634405,
-            "stdev": 0.0005429230982830213,
             "values": [
-                0.04083331298828125,
-                0.0396492805480957,
-                0.03877785491943359,
-                0.03871334457397461,
-                0.03868057632446289,
-                0.038803455352783206,
-                0.04250624084472656,
-                0.0422737922668457,
-                0.03878297424316406,
-                0.0376360969543457,
-                0.03747046279907226,
-                0.03753472137451172,
-                0.037720062255859374,
-                0.037548030853271484,
-                0.03764019012451172,
-                0.03959296035766602,
-                0.039024639129638675,
-                0.03762790298461914,
-                0.037493759155273435,
-                0.037577728271484374,
-                0.0373831672668457,
-                0.03742105484008789,
-                0.037416961669921874,
-                0.03785420989990235,
-                0.037594112396240234,
-                0.037510177612304685,
-                0.0375838737487793,
-                0.03751424026489258,
-                0.03747430419921875,
-                0.03755724716186523,
-                0.03748044967651367,
-                0.037501953125,
-                0.03742924880981445,
-                0.03751424026489258,
-                0.037493759155273435,
-                0.03752140808105469,
-                0.037631999969482424,
-                0.03749478530883789,
-                0.037585918426513674,
-                0.037539840698242184,
-                0.03801190567016602,
-                0.03746713638305664,
-                0.037438465118408204,
-                0.03745894241333008,
-                0.03748147201538086,
-                0.03748966217041016,
-                0.037531646728515625,
-                0.03762995147705078,
-                0.037571807861328126,
-                0.03756748962402344,
-                0.03741388702392578,
-                0.03745587158203125,
-                0.037408767700195314,
-                0.03791667175292969,
-                0.037465087890625,
-                0.03755724716186523,
-                0.03749289703369141,
-                0.03757363128662109,
-                0.03751321411132812,
-                0.037698558807373043,
-                0.03751436614990234,
-                0.03745177459716797,
-                0.03744473648071289,
-                0.03752140808105469,
-                0.03765555191040039,
-                0.03749587249755859,
-                0.037424129486083986,
-                0.037612545013427735,
-                0.03747654342651367,
-                0.03746815872192383,
-                0.03738214492797851,
-                0.037526527404785154,
-                0.03740671920776367,
-                0.037539840698242184,
-                0.03758899307250976,
-                0.037498878479003905,
-                0.037375999450683595,
-                0.0374835205078125,
-                0.037253120422363284,
-                0.037397502899169925,
-                0.03752140808105469,
-                0.03740262222290039,
-                0.03747148895263672,
-                0.03748659133911133,
-                0.03748454284667969,
-                0.037356544494628906,
-                0.03755417633056641,
-                0.037479423522949216,
-                0.03739033508300781,
-                0.037285888671875,
-                0.03739875030517578,
-                0.03747123336791992,
-                0.03744153594970703,
-                0.03727667236328125,
-                0.03795251083374023,
-                0.03746303939819336,
-                0.037449726104736326,
-                0.03740671920776367,
-                0.03774259185791016,
-                0.0375849609375,
-                0.03748966217041016,
-                0.03739340972900391,
-                0.03735551834106445,
-                0.03747020721435547,
-                0.03750707244873047,
-                0.037427425384521484,
-                0.03745894241333008,
-                0.03791999816894531,
-                0.03744768142700195,
-                0.03762073516845703,
-                0.03748659133911133,
-                0.037461280822753906,
-                0.037321727752685545,
-                0.03740979385375977,
-                0.03756972885131836,
-                0.03764019012451172,
-                0.03747430419921875,
-                0.03753676986694336,
-                0.03746201705932617,
-                0.03761356735229492,
-                0.03756748962402344,
-                0.038596607208251955,
-                0.03748966217041016,
-                0.037658687591552734,
-                0.037543201446533205,
-                0.03762278366088867,
-                0.037607425689697264,
-                0.037610496520996094,
-                0.03763507080078125,
-                0.037550079345703126,
-                0.037564414978027344,
-                0.037550079345703126,
-                0.03762688064575195,
-                0.037435489654541014,
-                0.03760844802856445,
-                0.03762073516845703,
-                0.03759820938110352,
-                0.03747635269165039,
-                0.03746713638305664,
-                0.03780198287963867,
-                0.037628929138183595,
-                0.03754393768310547,
-                0.037512191772460936,
-                0.037525505065917966,
-                0.03752140808105469,
-                0.03739852905273437,
-                0.037602302551269534,
-                0.03771187210083008,
-                0.03778662490844727,
-                0.03755321502685547,
-                0.0375551986694336,
-                0.03760025787353516,
-                0.03767091369628906,
-                0.03760947036743164,
-                0.037459968566894535,
-                0.03748659133911133,
-                0.03765862274169922,
-                0.03752243041992188,
-                0.03760332870483398,
-                0.03758796691894531,
-                0.037966846466064456,
-                0.03758694458007812,
-                0.03755212783813477,
-                0.037722110748291016,
-                0.03750707244873047,
-                0.03765248107910156,
-                0.037651454925537106,
-                0.03752447891235351,
-                0.037582847595214845,
-                0.03757056045532227,
-                0.03769241714477539,
-                0.03759820938110352,
-                0.03764121627807617,
-                0.03753881454467774,
-                0.03757878494262695,
-                0.037571582794189456,
-                0.03770675277709961,
-                0.03759001541137695,
-                0.03759513473510742,
-                0.037577728271484374,
-                0.03751321411132812,
-                0.03750809478759766,
-                0.03749683380126953,
-                0.03736576080322265,
-                0.03752959823608398,
-                0.03752959823608398,
-                0.03756748962402344,
-                0.037533695220947266,
-                0.03778355026245117,
-                0.037574657440185545,
-                0.037427200317382815,
-                0.03747430419921875,
-                0.037427200317382815,
-                0.037424129486083986,
-                0.03734233474731445,
-                0.03745382308959961,
-                0.0374466552734375,
-                0.03764121627807617,
-                0.037425151824951174,
-                0.03752345657348633,
-                0.0374917106628418,
-                0.03790643310546875,
-                0.037384193420410154,
-                0.037427200317382815,
-                0.03742822265625,
-                0.0374917106628418,
-                0.03751424026489258,
-                0.037548030853271484,
-                0.03739648056030274,
-                0.03751321411132812,
-                0.03755440139770508,
-                0.037579776763916016,
-                0.03755929565429687,
-                0.037591041564941405,
-                0.037835777282714846,
-                0.037550079345703126,
-                0.03752457427978516,
-                0.037572608947753904,
-                0.03750707244873047,
-                0.03764326477050781,
-                0.03754393768310547,
-                0.03741491317749023,
-                0.037429344177246096,
-                0.037440513610839846,
-                0.037408767700195314,
-                0.03741193771362305,
-                0.037364734649658206,
-                0.037393566131591796,
-                0.03759718322753906,
-                0.037493759155273435,
-                0.037466110229492186,
-                0.03753779220581055,
-                0.037440513610839846,
-                0.03744153594970703,
-                0.037465087890625,
-                0.037526527404785154,
-                0.03742627334594727,
-                0.03760435104370117,
-                0.03751833724975586,
-                0.037563392639160156,
-                0.03736678314208984,
-                0.037585918426513674,
-                0.03748659133911133,
-                0.03735859298706055,
-                0.037515262603759765,
-                0.037602302551269534,
-                0.0375203857421875,
-                0.037531871795654294,
-                0.037550079345703126,
-                0.03755929565429687,
-                0.037561344146728515,
-                0.03752959823608398,
-                0.037628929138183595,
-                0.03765964889526367,
-                0.03752959823608398,
-                0.03793203353881836,
-                0.037667839050292966,
-                0.03751424026489258,
-                0.03759206390380859,
-                0.03753062438964844,
-                0.03756851196289063,
-                0.03755724716186523,
-                0.03762073516845703,
-                0.03755110549926758,
-                0.037589118957519534,
-                0.03769241714477539
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 6802.378280466301
         },
         "energy": null,
         "efficiency": null
@@ -290,23 +57,22 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 991.27296,
-            "max_vram": 2858.418176,
-            "max_reserved": 2373.976064,
-            "max_allocated": 2326.085632
         },
         "latency": {
             "unit": "s",
-            "mean": 7.318287711143492,
-            "stdev": 0.003758425712581559,
             "values": [
-                7.322046136856073,
-                7.31452928543091
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 34.84421630646111
         },
         "energy": null,
         "efficiency": null
@@ -315,524 +81,269 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.028699167494680367,
-            "stdev": 0.00011730269864838747,
             "values": [
-                0.029384384155273436,
-                0.02918604850769043,
-                0.02877644729614258,
-                0.028843008041381835,
-                0.02877542304992676,
-                0.02872831916809082,
-                0.02874675178527832,
-                0.029063167572021483,
-                0.02921881675720215,
-                0.029207551956176758,
-                0.028906496047973632,
-                0.028685312271118164,
-                0.028679168701171875,
-                0.028735488891601563,
-                0.028726272583007813,
-                0.0287774715423584,
-                0.028890111923217773,
-                0.028852224349975586,
-                0.028719104766845704,
-                0.028703744888305665,
-                0.028848127365112306,
-                0.029024255752563476,
-                0.02877952003479004,
-                0.02880614471435547,
-                0.02876006317138672,
-                0.028825599670410155,
-                0.02878156852722168,
-                0.028655616760253907,
-                0.02874470329284668,
-                0.028888063430786134,
-                0.028821504592895508,
-                0.02873139190673828,
-                0.028710912704467774,
-                0.02870681571960449,
-                0.028647424697875977,
-                0.028654592514038086,
-                0.028767232894897462,
-                0.028709888458251953,
-                0.028799999237060548,
-                0.02876825523376465,
-                0.028955776214599608,
-                0.02864627265930176,
-                0.02874880027770996,
-                0.028682239532470705,
-                0.02875596809387207,
-                0.028837888717651368,
-                0.02893721580505371,
-                0.028835840225219726,
-                0.02877337646484375,
-                0.02876108741760254,
-                0.028898303985595702,
-                0.028799999237060548,
-                0.02894233512878418,
-                0.028688383102416993,
-                0.02919628715515137,
-                0.02878054428100586,
-                0.02872319984436035,
-                0.028759040832519532,
-                0.028810239791870116,
-                0.028725248336791992,
-                0.028832767486572267,
-                0.028658687591552736,
-                0.028824575424194337,
-                0.028836864471435547,
-                0.02867308807373047,
-                0.028733375549316407,
-                0.02870681571960449,
-                0.028729343414306642,
-                0.028664831161499024,
-                0.02877542304992676,
-                0.028712959289550782,
-                0.029117439270019533,
-                0.02875289535522461,
-                0.028820480346679687,
-                0.02875801658630371,
-                0.028694528579711914,
-                0.028794879913330077,
-                0.028832767486572267,
-                0.02875699234008789,
-                0.028791807174682618,
-                0.028706016540527343,
-                0.028621599197387694,
-                0.02856345558166504,
-                0.028799999237060548,
-                0.02874367904663086,
-                0.0286167049407959,
-                0.02871500778198242,
-                0.028659711837768553,
-                0.028602367401123048,
-                0.028643327713012694,
-                0.0285849609375,
-                0.028891136169433593,
-                0.028677120208740234,
-                0.02873855972290039,
-                0.028643327713012694,
-                0.028742656707763672,
-                0.028712959289550782,
-                0.028693504333496093,
-                0.028684352874755858,
-                0.02864531135559082,
-                0.028692480087280273,
-                0.028645376205444335,
-                0.02858700752258301,
-                0.028683263778686522,
-                0.028579839706420897,
-                0.028681215286254884,
-                0.028622848510742187,
-                0.028656639099121094,
-                0.028604415893554686,
-                0.028676095962524413,
-                0.02855219268798828,
-                0.028638208389282226,
-                0.028671199798583985,
-                0.028682016372680665,
-                0.028705791473388673,
-                0.02874470329284668,
-                0.028639232635498047,
-                0.028632064819335938,
-                0.028504064559936523,
-                0.02854195213317871,
-                0.028662784576416016,
-                0.028703744888305665,
-                0.02858598327636719,
-                0.028598272323608398,
-                0.028837888717651368,
-                0.028683263778686522,
-                0.028672000885009766,
-                0.028826623916625976,
-                0.028628992080688476,
-                0.028672000885009766,
-                0.028624895095825196,
-                0.028663808822631837,
-                0.028618751525878908,
-                0.028614656448364258,
-                0.02855936050415039,
-                0.028609535217285157,
-                0.0285296630859375,
-                0.028687360763549805,
-                0.028609535217285157,
-                0.028638208389282226,
-                0.028614656448364258,
-                0.029031423568725585,
-                0.028727296829223634,
-                0.028609535217285157,
-                0.028618751525878908,
-                0.028605440139770507,
-                0.028621824264526367,
-                0.02872422409057617,
-                0.028624895095825196,
-                0.028624895095825196,
-                0.028666879653930662,
-                0.028688383102416993,
-                0.028626943588256838,
-                0.028641279220581056,
-                0.02854911994934082,
-                0.028621824264526367,
-                0.028735488891601563,
-                0.028654592514038086,
-                0.02860339164733887,
-                0.028676095962524413,
-                0.028667903900146483,
-                0.028717056274414062,
-                0.02873139190673828,
-                0.028690431594848635,
-                0.028624895095825196,
-                0.028654592514038086,
-                0.028654592514038086,
-                0.028770303726196288,
-                0.02875289535522461,
-                0.028735488891601563,
-                0.028651647567749024,
-                0.028687231063842772,
-                0.02873958396911621,
-                0.028750848770141602,
-                0.0285614070892334,
-                0.028809215545654295,
-                0.028589216232299805,
-                0.02868515205383301,
-                0.0286167049407959,
-                0.028661760330200195,
-                0.028672000885009766,
-                0.02873958396911621,
-                0.02881331253051758,
-                0.02878156852722168,
-                0.02857369613647461,
-                0.02855936050415039,
-                0.028695552825927735,
-                0.028753919601440428,
-                0.028658687591552736,
-                0.028676095962524413,
-                0.028646400451660156,
-                0.028820480346679687,
-                0.02873651123046875,
-                0.02876006317138672,
-                0.028647424697875977,
-                0.029197311401367186,
-                0.02854604721069336,
-                0.028712959289550782,
-                0.028491775512695314,
-                0.028677120208740234,
-                0.028614656448364258,
-                0.02860032081604004,
-                0.028717056274414062,
-                0.028617727279663087,
-                0.028645376205444335,
-                0.02878156852722168,
-                0.0286167049407959,
-                0.02857062339782715,
-                0.028690528869628907,
-                0.02871900749206543,
-                0.02858700752258301,
-                0.02878873634338379,
-                0.028786687850952147,
-                0.02873958396911621,
-                0.028633087158203126,
-                0.028621824264526367,
-                0.028590080261230468,
-                0.028815359115600587,
-                0.028633087158203126,
-                0.028710912704467774,
-                0.028664831161499024,
-                0.028645376205444335,
-                0.02856959915161133,
-                0.028625919342041017,
-                0.028648448944091798,
-                0.028718080520629883,
-                0.028618751525878908,
-                0.028677120208740234,
-                0.028669952392578125,
-                0.028729343414306642,
-                0.028668928146362304,
-                0.028712959289550782,
-                0.028633087158203126,
-                0.028612607955932616,
-                0.028645376205444335,
-                0.028650495529174806,
-                0.02874060821533203,
-                0.028737535476684572,
-                0.02857369613647461,
-                0.028817407608032225,
-                0.028645376205444335,
-                0.028682239532470705,
-                0.028688383102416993,
-                0.02882252883911133,
-                0.028712959289550782,
-                0.028655616760253907,
-                0.028528640747070313,
-                0.028660736083984374,
-                0.028589088439941405,
-                0.028756959915161133,
-                0.028620800018310546,
-                0.028669952392578125,
-                0.02855241584777832,
-                0.028652320861816405,
-                0.02857164764404297,
-                0.0287457275390625,
-                0.02872012710571289,
-                0.02869964790344238,
-                0.028674047470092775,
-                0.02875596809387207,
-                0.028625919342041017,
-                0.028673023223876954,
-                0.02869964790344238,
-                0.028685312271118164,
-                0.029190143585205077,
-                0.02873139190673828,
-                0.02878361511230469,
-                0.028750848770141602,
-                0.028676095962524413,
-                0.028666879653930662,
-                0.028667903900146483,
-                0.028694528579711914,
-                0.02871603202819824,
-                0.028678144454956055,
-                0.02875494384765625,
-                0.028651519775390624,
-                0.02875801658630371,
-                0.02874675178527832,
-                0.02877235221862793,
-                0.02873139190673828,
-                0.028992511749267577,
-                0.02875187110900879,
-                0.028684288024902343,
-                0.028785663604736327,
-                0.028711936950683595,
-                0.02872012710571289,
-                0.028865535736083983,
-                0.028841983795166014,
-                0.02877132797241211,
-                0.02860851287841797,
-                0.028832767486572267,
-                0.028737535476684572,
-                0.02878054428100586,
-                0.02872831916809082,
-                0.028831743240356447,
-                0.028669952392578125,
-                0.028711936950683595,
-                0.028666879653930662,
-                0.028807167053222657,
-                0.028737535476684572,
-                0.02872012710571289,
-                0.028628992080688476,
-                0.028835840225219726,
-                0.028676095962524413,
-                0.02875187110900879,
-                0.028712959289550782,
-                0.028827648162841796,
-                0.02876620864868164,
-                0.028710912704467774,
-                0.028677120208740234,
-                0.02880512046813965,
-                0.028727296829223634,
-                0.028770303726196288,
-                0.028725248336791992,
-                0.028767232894897462,
-                0.028718080520629883,
-                0.028677120208740234,
-                0.02875699234008789,
-                0.02876927947998047,
-                0.028742656707763672,
-                0.02877132797241211,
-                0.028607519149780273,
-                0.02867299270629883,
-                0.02870681571960449,
-                0.0285849609375,
-                0.028579839706420897,
-                0.028625919342041017,
-                0.028697599411010744,
-                0.028687360763549805,
-                0.028655616760253907,
-                0.028658687591552736,
-                0.028718080520629883,
-                0.028656639099121094,
-                0.028641279220581056,
-                0.02901299285888672,
-                0.028620927810668946,
-                0.028702592849731444,
-                0.028696575164794923,
-                0.028727296829223634,
-                0.028709888458251953,
-                0.028636159896850585,
-                0.028597248077392577,
-                0.028665855407714845,
-                0.028678144454956055,
-                0.028799999237060548,
-                0.02860748863220215,
-                0.028669952392578125,
-                0.028703744888305665,
-                0.028662784576416016,
-                0.028659711837768553,
-                0.028679168701171875,
-                0.028997856140136717,
-                0.028719903945922852,
-                0.028668928146362304,
-                0.02859519958496094,
-                0.028630016326904296,
-                0.028618751525878908,
-                0.02857779121398926,
-                0.028632064819335938,
-                0.028484607696533205,
-                0.028670976638793946,
-                0.02857676887512207,
-                0.028695552825927735,
-                0.028656639099121094,
-                0.028733440399169922,
-                0.02851430320739746,
-                0.02869964790344238,
-                0.028594175338745118,
-                0.028679168701171875,
-                0.028630016326904296,
-                0.02859110450744629,
-                0.028503040313720703,
-                0.028614656448364258,
-                0.028515327453613282,
-                0.028645376205444335,
-                0.02855219268798828,
-                0.028673023223876954,
-                0.028508159637451173,
-                0.028762111663818358,
-                0.028620800018310546,
-                0.028622848510742187,
-                0.028635135650634767,
-                0.028590080261230468,
-                0.028717056274414062,
-                0.028684288024902343,
-                0.028622848510742187,
-                0.02874675178527832,
-                0.028626943588256838,
-                0.028579839706420897,
-                0.028641279220581056,
-                0.02872012710571289,
-                0.028663808822631837,
-                0.02874367904663086,
-                0.028672000885009766,
-                0.028701696395874023,
-                0.02859929656982422,
-                0.0287324161529541,
-                0.028622848510742187,
-                0.0285665283203125,
-                0.028665855407714845,
-                0.02856550407409668,
-                0.02856550407409668,
-                0.02869862365722656,
-                0.028794879913330077,
-                0.029240320205688477,
-                0.028662784576416016,
-                0.02856857681274414,
-                0.028627967834472655,
-                0.028653568267822265,
-                0.028625919342041017,
-                0.028630016326904296,
-                0.028719104766845704,
-                0.028672000885009766,
-                0.028611583709716795,
-                0.028639232635498047,
-                0.028697599411010744,
-                0.028703744888305665,
-                0.028636159896850585,
-                0.028750848770141602,
-                0.02857062339782715,
-                0.028703744888305665,
-                0.02898636817932129,
-                0.028638208389282226,
-                0.0286167049407959,
-                0.028662784576416016,
-                0.028596223831176756,
-                0.028589056015014647,
-                0.028511232376098632,
-                0.028598272323608398,
-                0.028677120208740234,
-                0.0287774715423584,
-                0.028691455841064452,
-                0.02874777603149414,
-                0.02862816047668457,
-                0.028746623992919922,
-                0.02870675277709961,
-                0.02873958396911621,
-                0.02875494384765625,
-                0.028853248596191407,
-                0.02879692840576172,
-                0.028817407608032225,
-                0.028742656707763672,
-                0.028688383102416993,
-                0.028660736083984374,
-                0.028618751525878908,
-                0.028712959289550782,
-                0.028596223831176756,
-                0.028832767486572267,
-                0.02852659225463867,
-                0.028602367401123048,
-                0.028690431594848635,
-                0.02866694450378418,
-                0.028684223175048828,
-                0.02854707145690918,
-                0.028622848510742187,
-                0.028528640747070313,
-                0.028823551177978517,
-                0.028889087677001952,
-                0.02874163246154785,
-                0.028661760330200195,
-                0.0287324161529541,
-                0.028601343154907227,
-                0.028684288024902343,
-                0.02858620834350586,
-                0.028834592819213866,
-                0.02859212875366211,
-                0.028665855407714845,
-                0.028490751266479493,
-                0.028670976638793946,
-                0.028564479827880858,
-                0.028684288024902343,
-                0.028645376205444335,
-                0.028795904159545898,
-                0.0286167049407959,
-                0.028931072235107422,
-                0.028642303466796876,
-                0.02874060821533203,
-                0.028516351699829103,
-                0.02858188819885254,
-                0.02858393669128418,
-                0.028686336517333984,
-                0.028685312271118164,
-                0.028697599411010744,
-                0.028601343154907227,
-                0.028672000885009766,
-                0.028453887939453124,
-                0.028610559463500978,
-                0.02858598327636719,
-                0.02872217559814453,
-                0.028505088806152344,
-                0.02862387275695801,
-                0.02916761589050293,
-                0.02872012710571289,
-                0.028468223571777345,
-                0.02871603202819824,
-                0.02852454376220703,
-                0.02873651123046875,
-                0.028590080261230468,
-                0.028719104766845704,
-                0.02856857681274414,
-                0.028613632202148437,
-                0.02852659225463867,
-                0.028680192947387696,
-                0.02860256004333496,
-                0.0287076473236084,
-                0.02853068733215332,
-                0.028619775772094725,
-                0.028624895095825196,
-                0.028624895095825196,
-                0.028602367401123048
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 34.8442163064611
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 3078.4512,
+            "max_vram": 5981.077504,
+            "max_reserved": 5496.635392,
+            "max_allocated": 5273.977856
         },
         "latency": {
             "unit": "s",
+            "mean": 0.30716638276071256,
+            "stdev": 0.0009185968346906661,
             "values": [
+                0.3119903869628906,
+                0.3088045959472656,
+                0.3071365661621094,
+                0.30677197265625,
+                0.30710476684570315,
+                0.30677401733398435,
+                0.30709042358398436,
+                0.30680267333984373,
+                0.30704742431640625,
+                0.306798583984375,
+                0.30709051513671876,
+                0.30681396484375,
+                0.30705255126953124,
+                0.3068487548828125,
+                0.3070996398925781,
+                0.30679653930664064,
+                0.30710372924804685,
+                0.30680294799804686,
+                0.3070330810546875,
+                0.3068620910644531,
+                0.3070586853027344,
+                0.3068264465332031,
+                0.30706277465820314,
+                0.30687640380859377,
+                0.3070453796386719,
+                0.30687948608398435,
+                0.3071150207519531,
+                0.3068538818359375,
+                0.30714572143554686,
+                0.306808837890625,
+                0.3070709838867188,
+                0.30686822509765627,
+                0.3070535583496094
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 833.4245359116268
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 3078.4512,
+            "max_vram": 6234.832896,
+            "max_reserved": 5750.390784,
+            "max_allocated": 5425.04704
         },
         "latency": {
             "unit": "s",
+            "mean": 76.54886825561519,
+            "stdev": 0,
             "values": [
+                76.54886825561519
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 3.331205356929554
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.3001916402180988,
+            "stdev": 0.00047061196976397096,
             "values": [
+                0.29955584716796874,
+                0.3000914001464844,
+                0.3001548767089844,
+                0.29994189453125,
+                0.3001149291992187,
+                0.30003814697265624,
+                0.30010983276367187,
+                0.300147705078125,
+                0.3001466979980469,
+                0.3000217590332031,
+                0.3000340576171875,
+                0.3000115051269531,
+                0.300105712890625,
+                0.3000074157714844,
+                0.3001026611328125,
+                0.30018865966796876,
+                0.2999521179199219,
+                0.2999582824707031,
+                0.3000924072265625,
+                0.30001254272460937,
+                0.3002030029296875,
+                0.2999336853027344,
+                0.29999819946289064,
+                0.2998835144042969,
+                0.3000094604492187,
+                0.3000657958984375,
+                0.2999941101074219,
+                0.30008935546875,
+                0.3000012817382812,
+                0.29993472290039064,
+                0.30010366821289064,
+                0.29994085693359374,
+                0.30014361572265624,
+                0.2999797668457031,
+                0.30006500244140627,
+                0.29997955322265624,
+                0.3001272277832031,
+                0.29998797607421873,
+                0.30003302001953125,
+                0.3002931213378906,
+                0.30011700439453126,
+                0.3000688781738281,
+                0.30010366821289064,
+                0.300126220703125,
+                0.3000361022949219,
+                0.30014566040039065,
+                0.3000084533691406,
+                0.3001968688964844,
+                0.3000565795898438,
+                0.3001815185546875,
+                0.30014566040039065,
+                0.3001845703125,
+                0.30017230224609376,
+                0.3000637512207031,
+                0.3001221008300781,
+                0.300037109375,
+                0.30008319091796876,
+                0.3000627136230469,
+                0.30003201293945314,
+                0.3001763916015625,
+                0.3001343994140625,
+                0.3002296447753906,
+                0.30043136596679687,
+                0.30008218383789065,
+                0.30005966186523436,
+                0.30004019165039064,
+                0.3001070861816406,
+                0.3002006530761719,
+                0.3000074157714844,
+                0.3001917419433594,
+                0.3000555419921875,
+                0.30009344482421874,
+                0.3000975341796875,
+                0.30015179443359374,
+                0.3001395263671875,
+                0.30008831787109375,
+                0.300084228515625,
+                0.300000244140625,
+                0.30001254272460937,
+                0.30010983276367187,
+                0.3000012817382812,
+                0.30006988525390627,
+                0.3000135803222656,
+                0.30007601928710936,
+                0.2999808044433594,
+                0.30020095825195314,
+                0.30154547119140623,
+                0.300253173828125,
+                0.3001047058105469,
+                0.30017535400390627,
+                0.30011117553710936,
+                0.3001391906738281,
+                0.3001538696289062,
+                0.30010983276367187,
+                0.3002378234863281,
+                0.30011187744140627,
+                0.30008831787109375,
+                0.3000862731933594,
+                0.3000516967773437,
+                0.3001136779785156,
+                0.3000975341796875,
+                0.3001600036621094,
+                0.3000924072265625,
+                0.299947021484375,
+                0.2999183349609375,
+                0.3000309753417969,
+                0.30010983276367187,
+                0.30004736328125,
+                0.30563739013671876,
+                0.30018865966796876,
+                0.3001395263671875,
+                0.3001917419433594,
+                0.3000719299316406,
+                0.3001845703125,
+                0.3001294250488281,
+                0.30013116455078126,
+                0.30000537109375,
+                0.3001128845214844,
+                0.30006784057617186,
+                0.3002572937011719,
+                0.3000975341796875,
+                0.3001702270507812,
+                0.3001200561523438,
+                0.3001395263671875,
+                0.3000238037109375,
+                0.30023373413085935,
+                0.3000985717773437,
+                0.30015896606445314,
+                0.3005962219238281,
+                0.3003504638671875,
+                0.300052490234375,
+                0.3002798156738281,
+                0.30012313842773436,
+                0.30017535400390627,
+                0.30045901489257815,
+                0.3001548767089844,
+                0.30305484008789063,
+                0.3030456237792969,
+                0.3004682312011719,
+                0.30018765258789065,
+                0.30012518310546876,
+                0.30031155395507814,
+                0.30003814697265624,
+                0.3001241455078125,
+                0.2999439392089844,
+                0.30018661499023436,
+                0.300084228515625,
+                0.3000668029785156,
+                0.30013644409179685,
+                0.3020902404785156,
+                0.3002941589355469,
+                0.30020913696289064,
+                0.3001077880859375,
+                0.3004375,
+                0.3000709228515625,
+                0.3002245178222656,
+                0.3001630859375,
+                0.30010162353515624,
+                0.30039141845703127,
+                0.30006170654296876,
+                0.3000852355957031,
+                0.3000924072265625,
+                0.3000227966308594,
+                0.2999931030273438,
+                0.30006170654296876,
+                0.3000555419921875,
+                0.30010879516601563,
+                0.300115966796875,
+                0.30013748168945314,
+                0.3001640930175781,
+                0.30014361572265624,
+                0.2999459838867187,
+                0.3002265625,
+                0.30009036254882815,
+                0.30015179443359374,
+                0.3001026611328125,
+                0.30017535400390627,
+                0.30015078735351564,
+                0.3001559143066406,
+                0.3001978759765625,
+                0.3002818603515625,
+                0.30021630859375,
+                0.3002941589355469,
+                0.3001835632324219,
+                0.3001743469238281,
+                0.3001825256347656,
+                0.30012518310546876,
+                0.30023065185546877,
+                0.30016717529296877,
+                0.30008319091796876,
+                0.3000657958984375,
+                0.29989376831054687,
+                0.30003302001953125,
+                0.300189697265625,
+                0.3000157775878906,
+                0.30008099365234375,
+                0.30001254272460937,
+                0.30023577880859376,
+                0.30006784057617186,
+                0.3001681823730469,
+                0.3006085205078125,
+                0.300199951171875,
+                0.3002019958496094,
+                0.30002688598632815,
+                0.3001077880859375,
+                0.3001190490722656,
+                0.3000606689453125,
+                0.3000606689453125,
+                0.3000791015625,
+                0.3000627136230469,
+                0.30002688598632815,
+                0.30010879516601563,
+                0.29994906616210937,
+                0.30015078735351564,
+                0.29995928955078127,
+                0.3001692199707031,
+                0.3001538696289062,
+                0.30012313842773436,
+                0.30022991943359373,
+                0.30020272827148436,
+                0.30016717529296877,
+                0.30015078735351564,
+                0.3002470397949219,
+                0.3000719299316406,
+                0.3001702270507812,
+                0.30023373413085935,
+                0.3001405334472656,
+                0.30029208374023436,
+                0.3001845703125,
+                0.30018560791015625,
+                0.3003924560546875,
+                0.30023678588867186,
+                0.3009228820800781,
+                0.30029208374023436,
+                0.3003290710449219,
+                0.30029815673828125,
+                0.3002378234863281,
+                0.30027059936523437,
+                0.30020404052734373,
+                0.30021221923828123,
+                0.3001958312988281,
+                0.30032589721679687,
+                0.30022467041015627,
+                0.3003216247558594,
+                0.3003056945800781,
+                0.30029180908203124,
+                0.30029107666015625,
+                0.30008218383789065,
+                0.30000436401367186,
+                0.3002746887207031,
+                0.3003036804199219,
+                0.3002559509277344,
+                0.3001610107421875,
+                0.30046414184570314,
+                0.3003453369140625
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 3.3312053569295537
         },
         "energy": null,
         "efficiency": null