kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Apr 16

Commit

112a0c2

•

1 Parent(s): 2bc4ab4

Upload /Mistral-7B-v0.1/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Mistral-7B-v0.1/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +430 -696

Mistral-7B-v0.1/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,194 +2,184 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 2520.65792,
             "max_vram": 5490.343936,
             "max_reserved": 5005.901824,
             "max_allocated": 4844.946944
         },
         "latency": {
             "unit": "s",
-            "mean": 0.05782990103925586,
-            "stdev": 0.0005964412042560866,
             "values": [
-                0.06358777618408203,
-                0.06096588897705078,
-                0.05835059356689453,
-                0.05796454238891602,
-                0.06010879898071289,
-                0.05883596801757812,
-                0.05797478485107422,
-                0.057490432739257816,
-                0.058055679321289064,
-                0.057474048614501956,
-                0.05794508743286133,
-                0.057530399322509765,
-                0.058011646270751956,
-                0.057504768371582034,
-                0.05803519821166992,
-                0.05745471954345703,
-                0.058103809356689455,
-                0.05746585464477539,
-                0.058008575439453126,
-                0.05751603317260742,
-                0.05798297500610351,
-                0.05747711944580078,
-                0.05790617752075195,
-                0.05746585464477539,
-                0.05800755310058594,
-                0.057613311767578126,
-                0.05802905654907226,
-                0.057499870300292966,
-                0.0579317741394043,
-                0.057447425842285155,
-                0.05799628829956055,
-                0.057475070953369144,
-                0.05798297500610351,
-                0.05747097778320313,
-                0.05802598571777344,
-                0.05750886535644531,
-                0.057940990447998046,
-                0.05746790313720703,
-                0.057981952667236325,
-                0.05747945785522461,
-                0.05788671875,
-                0.05744144058227539,
-                0.057985023498535154,
-                0.05744435119628906,
-                0.05794406509399414,
-                0.057447425842285155,
-                0.05793382263183594,
-                0.05748223876953125,
-                0.05795635223388672,
-                0.05742601776123047,
-                0.05796761703491211,
-                0.05755392074584961,
-                0.057981952667236325,
-                0.05740851211547852,
-                0.05795123291015625,
-                0.05747097778320313,
-                0.05796761703491211,
-                0.05746905517578125,
-                0.05791846466064453,
-                0.057673728942871094,
-                0.0579317741394043,
-                0.05744876861572266,
-                0.058031105041503904,
-                0.05749657440185547,
-                0.058363903045654295,
-                0.05752627182006836,
-                0.05801062393188477,
-                0.0574730224609375,
-                0.0579317741394043,
-                0.05747817611694336,
-                0.05791846466064453,
-                0.05748534393310547,
-                0.05795840072631836,
-                0.05748428726196289,
-                0.05795328140258789,
-                0.05747711944580078,
-                0.0579317741394043,
-                0.05742182540893555,
-                0.05800447845458984,
-                0.05746380615234375,
-                0.05791743850708008,
-                0.057507839202880856,
-                0.05794713592529297,
-                0.05748428726196289,
-                0.057913345336914064,
-                0.05747711944580078,
-                0.05811404800415039,
-                0.05752115249633789,
-                0.05795123291015625,
-                0.05747609710693359,
-                0.05806182479858398,
-                0.057431041717529295,
-                0.058003551483154295,
-                0.057504768371582034,
-                0.05822873687744141,
-                0.05754470443725586,
-                0.057981952667236325,
-                0.05751193618774414,
-                0.057957374572753906,
-                0.057504768371582034,
-                0.05802598571777344,
-                0.05748428726196289,
-                0.05797785568237305,
-                0.05748428726196289,
-                0.05798912048339844,
-                0.05749248123168945,
-                0.05801369476318359,
-                0.057491455078125,
-                0.0580055046081543,
-                0.05748735809326172,
-                0.05794815826416016,
-                0.05745356750488281,
-                0.057985023498535154,
-                0.05748121643066406,
-                0.05798092651367188,
-                0.05744947052001953,
-                0.05799935913085937,
-                0.05751910400390625,
-                0.058054656982421876,
-                0.05749862289428711,
-                0.05798912048339844,
-                0.057724929809570315,
-                0.058025054931640625,
-                0.05746380615234375,
-                0.05796147155761719,
-                0.05753753662109375,
-                0.058057727813720705,
-                0.05752217483520508,
-                0.05796044921875,
-                0.05764198303222656,
-                0.05800960159301758,
-                0.05755904006958008,
-                0.05798096084594727,
-                0.05750374221801758,
-                0.05801369476318359,
-                0.057490432739257816,
-                0.05797683334350586,
-                0.05748633575439453,
-                0.05793088150024414,
-                0.05748553466796875,
-                0.058016769409179686,
-                0.05753548812866211,
-                0.058022911071777344,
-                0.057499870300292966,
-                0.058057727813720705,
-                0.057475070953369144,
-                0.05803007888793945,
-                0.05758771133422851,
-                0.057985023498535154,
-                0.05752115249633789,
-                0.05801574325561523,
-                0.05750681686401367,
-                0.05796454238891602,
-                0.05759078216552734,
-                0.05802598571777344,
-                0.05755084609985352,
-                0.05821440124511719,
-                0.057540607452392575,
-                0.05798400115966797,
-                0.057490432739257816,
-                0.05804646301269531,
-                0.057545726776123046,
-                0.05795644760131836,
-                0.057491455078125,
-                0.058218494415283206,
-                0.057534465789794924,
-                0.05807308959960938,
-                0.05757952117919922,
-                0.058028030395507815,
-                0.05753548812866211,
-                0.05803212738037109,
-                0.05754982376098633,
-                0.05800447845458984
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 4426.775688691273
         },
         "energy": null,
         "efficiency": null
@@ -197,23 +187,22 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 2521.706496,
             "max_vram": 5710.544896,
             "max_reserved": 5226.102784,
             "max_allocated": 4911.274496
         },
         "latency": {
             "unit": "s",
-            "mean": 8.440941043853758,
-            "stdev": 0.011413997650144836,
             "values": [
-                8.452355041503903,
-                8.429527046203614
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 30.20990179592326
         },
         "energy": null,
         "efficiency": null
@@ -222,524 +211,269 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.03310172958374023,
-            "stdev": 0.00021079061247777177,
             "values": [
-                0.03572844696044922,
-                0.035100608825683596,
-                0.03332723236083984,
-                0.033161407470703126,
-                0.03317523193359375,
-                0.03329228973388672,
-                0.03314790344238281,
-                0.033102848052978515,
-                0.033140735626220705,
-                0.0331387825012207,
-                0.0332327995300293,
-                0.033122337341308594,
-                0.03316016006469726,
-                0.03321548843383789,
-                0.03321651077270508,
-                0.03313151931762695,
-                0.033123615264892575,
-                0.03307388687133789,
-                0.03308544158935547,
-                0.03306291198730469,
-                0.033051647186279294,
-                0.033107967376708985,
-                0.03318374252319336,
-                0.03298406219482422,
-                0.03306496047973633,
-                0.032949249267578126,
-                0.03324415969848633,
-                0.03315507125854492,
-                0.033086463928222655,
-                0.033476608276367184,
-                0.03312639999389649,
-                0.03313663864135742,
-                0.03316838455200195,
-                0.03322470474243164,
-                0.03312947082519531,
-                0.033007614135742186,
-                0.033124351501464845,
-                0.0332492790222168,
-                0.03316428756713867,
-                0.033317886352539065,
-                0.033315040588378905,
-                0.033176353454589844,
-                0.03321036911010742,
-                0.033285247802734376,
-                0.033796993255615235,
-                0.033329151153564454,
-                0.03318067169189453,
-                0.03320729446411133,
-                0.03332505416870117,
-                0.03320012664794922,
-                0.03318374252319336,
-                0.03324415969848633,
-                0.03316438293457031,
-                0.033195934295654296,
-                0.03311820983886719,
-                0.03318374252319336,
-                0.03317452621459961,
-                0.03331379318237305,
-                0.03316249465942383,
-                0.03321932983398437,
-                0.033314815521240236,
-                0.03326976013183594,
-                0.033185791015625,
-                0.033154048919677735,
-                0.03321343994140625,
-                0.03322163009643555,
-                0.03335168075561523,
-                0.03324006271362305,
-                0.033123390197753906,
-                0.03322463989257812,
-                0.033182880401611325,
-                0.03321225738525391,
-                0.03322675323486328,
-                0.03322367858886719,
-                0.033095745086669924,
-                0.033196990966796874,
-                0.03314995193481445,
-                0.0331960334777832,
-                0.033209598541259766,
-                0.033299198150634764,
-                0.03322470474243164,
-                0.03321241760253906,
-                0.03313663864135742,
-                0.03336294555664063,
-                0.033037441253662106,
-                0.03419123077392578,
-                0.03421500778198242,
-                0.033410976409912106,
-                0.033175743103027344,
-                0.03322457504272461,
-                0.033476768493652345,
-                0.03319180679321289,
-                0.033048480987548826,
-                0.03319091033935547,
-                0.033099777221679685,
-                0.03322473526000977,
-                0.03324105453491211,
-                0.03316326522827148,
-                0.03314176177978516,
-                0.033277950286865234,
-                0.03313868713378906,
-                0.03322272109985352,
-                0.03307513427734375,
-                0.033260543823242186,
-                0.03368038558959961,
-                0.03302297592163086,
-                0.033067008972167966,
-                0.033122337341308594,
-                0.033328094482421876,
-                0.033113086700439456,
-                0.03310079956054687,
-                0.033173503875732424,
-                0.0330967025756836,
-                0.03319529724121094,
-                0.033060577392578124,
-                0.03325459289550781,
-                0.033119041442871096,
-                0.03300454330444336,
-                0.03299532699584961,
-                0.03313049697875976,
-                0.03316326522827148,
-                0.032993377685546874,
-                0.03323587036132813,
-                0.03308441543579101,
-                0.03307443237304687,
-                0.033071872711181644,
-                0.03312134552001953,
-                0.032997310638427736,
-                0.033071392059326174,
-                0.0330206413269043,
-                0.03294230270385742,
-                0.03292649459838867,
-                0.03303936004638672,
-                0.03300249481201172,
-                0.03306393432617188,
-                0.03303123092651367,
-                0.0329420166015625,
-                0.03299343872070312,
-                0.033041248321533205,
-                0.033091583251953126,
-                0.03301375961303711,
-                0.03321654510498047,
-                0.03327587127685547,
-                0.032876544952392575,
-                0.033081344604492184,
-                0.03303628921508789,
-                0.03301068878173828,
-                0.03292876815795898,
-                0.03303644943237305,
-                0.03296649551391601,
-                0.03305984115600586,
-                0.033483966827392575,
-                0.033032222747802736,
-                0.032884513854980466,
-                0.032895999908447264,
-                0.032985183715820314,
-                0.032986015319824216,
-                0.032955390930175785,
-                0.032912384033203124,
-                0.033081344604492184,
-                0.03307724761962891,
-                0.03327199935913086,
-                0.033079105377197264,
-                0.03313779067993164,
-                0.033027008056640626,
-                0.033469375610351564,
-                0.03287756729125976,
-                0.03308748626708984,
-                0.03295235061645508,
-                0.033172447204589846,
-                0.03306393432617188,
-                0.0331141128540039,
-                0.032939006805419925,
-                0.03323289489746094,
-                0.03303535842895508,
-                0.03311196899414062,
-                0.032935935974121096,
-                0.033086463928222655,
-                0.03290828704833984,
-                0.032909313201904294,
-                0.03294630432128906,
-                0.032944000244140625,
-                0.032927745819091796,
-                0.03303760147094727,
-                0.03321929550170898,
-                0.033099777221679685,
-                0.033108192443847655,
-                0.03321628952026367,
-                0.03309363174438477,
-                0.03308236694335937,
-                0.03294416046142578,
-                0.0331366081237793,
-                0.03298406219482422,
-                0.03304550552368164,
-                0.033030208587646485,
-                0.03293689727783203,
-                0.03298099136352539,
-                0.03285094451904297,
-                0.03304457473754883,
-                0.0329901123046875,
-                0.033124351501464845,
-                0.03314191818237305,
-                0.03306275177001953,
-                0.03311513519287109,
-                0.03307212829589844,
-                0.033007614135742186,
-                0.03311513519287109,
-                0.033165313720703124,
-                0.033040382385253905,
-                0.03298611068725586,
-                0.03293491363525391,
-                0.03302822494506836,
-                0.03345292663574219,
-                0.03327907180786133,
-                0.03291638565063477,
-                0.033116161346435545,
-                0.032914432525634765,
-                0.033037311553955076,
-                0.03298918533325195,
-                0.033058815002441407,
-                0.032919551849365236,
-                0.033051647186279294,
-                0.03299964904785156,
-                0.03303094482421875,
-                0.03297075271606445,
-                0.03306086349487305,
-                0.03337420654296875,
-                0.033068031311035154,
-                0.0329431037902832,
-                0.03307827377319336,
-                0.03304857635498047,
-                0.03309372711181641,
-                0.03320412826538086,
-                0.033113086700439456,
-                0.033181697845458984,
-                0.03314710235595703,
-                0.033155902862548825,
-                0.03309580612182617,
-                0.03299833679199219,
-                0.03306691360473633,
-                0.03307417678833008,
-                0.0330885124206543,
-                0.03295654296875,
-                0.033167232513427736,
-                0.03298611068725586,
-                0.03306601715087891,
-                0.03310895919799805,
-                0.03291033554077148,
-                0.03291340637207031,
-                0.033197246551513675,
-                0.033151809692382815,
-                0.03296672058105469,
-                0.03305161666870117,
-                0.03302601623535156,
-                0.03341107177734375,
-                0.0331673583984375,
-                0.03307724761962891,
-                0.033185791015625,
-                0.033075199127197266,
-                0.032939006805419925,
-                0.03308031845092774,
-                0.0333240966796875,
-                0.03308230209350586,
-                0.03307929611206055,
-                0.03302604675292969,
-                0.03306086349487305,
-                0.03299225616455078,
-                0.033046817779541014,
-                0.0331363525390625,
-                0.033320960998535154,
-                0.03310079956054687,
-                0.033565696716308595,
-                0.033091808319091795,
-                0.033137439727783206,
-                0.03304959869384766,
-                0.033058815002441407,
-                0.033068031311035154,
-                0.03316633605957031,
-                0.0330618896484375,
-                0.03309081649780273,
-                0.03296128082275391,
-                0.03320832061767578,
-                0.03308441543579101,
-                0.0332861442565918,
-                0.03323904037475586,
-                0.03363942337036133,
-                0.033037311553955076,
-                0.03306598281860352,
-                0.03304579162597656,
-                0.03302883148193359,
-                0.03300966262817383,
-                0.0331776008605957,
-                0.032996353149414064,
-                0.033170433044433595,
-                0.033067230224609376,
-                0.03310160064697266,
-                0.03306528091430664,
-                0.033051326751708986,
-                0.03304345703125,
-                0.033067008972167966,
-                0.033113086700439456,
-                0.03313663864135742,
-                0.03306496047973633,
-                0.03307628631591797,
-                0.03310688018798828,
-                0.0332042236328125,
-                0.033149185180664065,
-                0.03320499038696289,
-                0.033081375122070315,
-                0.03317679977416992,
-                0.03298892974853516,
-                0.03305472183227539,
-                0.03310182571411133,
-                0.03306496047973633,
-                0.03302707290649414,
-                0.03310182571411133,
-                0.03304652786254883,
-                0.03307622528076172,
-                0.03299737548828125,
-                0.033175552368164066,
-                0.0331776008605957,
-                0.0331141128540039,
-                0.03305267333984375,
-                0.03300966262817383,
-                0.033143806457519534,
-                0.03303734588623047,
-                0.03301801681518555,
-                0.03318764877319336,
-                0.03318067169189453,
-                0.03320217514038086,
-                0.033108192443847655,
-                0.03315894317626953,
-                0.03373673629760742,
-                0.03318268966674805,
-                0.033173503875732424,
-                0.033205249786376956,
-                0.03317891311645508,
-                0.03320291137695312,
-                0.03308236694335937,
-                0.0331069450378418,
-                0.03297814559936523,
-                0.033092384338378904,
-                0.032952320098876955,
-                0.03308441543579101,
-                0.033097793579101566,
-                0.033175487518310544,
-                0.03356159973144531,
-                0.03320012664794922,
-                0.03310182571411133,
-                0.03315814590454102,
-                0.03322060775756836,
-                0.0331673583984375,
-                0.03310079956054687,
-                0.033051647186279294,
-                0.03307334518432617,
-                0.033067840576171875,
-                0.0331409912109375,
-                0.0331220474243164,
-                0.033089534759521484,
-                0.033050624847412106,
-                0.033097728729248044,
-                0.033023998260498046,
-                0.03313356781005859,
-                0.032981056213378906,
-                0.03310995101928711,
-                0.03301171112060547,
-                0.03317046356201172,
-                0.03301577758789063,
-                0.03308348846435547,
-                0.033062816619873044,
-                0.03309283065795898,
-                0.03302889633178711,
-                0.03303936004638672,
-                0.03303424072265625,
-                0.03306905746459961,
-                0.03305779266357422,
-                0.03301171112060547,
-                0.03303219223022461,
-                0.03308748626708984,
-                0.03295129776000977,
-                0.03302912139892578,
-                0.03304652786254883,
-                0.0329238395690918,
-                0.033054527282714845,
-                0.03309363174438477,
-                0.032879615783691404,
-                0.032865409851074216,
-                0.0330463981628418,
-                0.032925918579101564,
-                0.03297257614135742,
-                0.03290726470947265,
-                0.03301593780517578,
-                0.032984031677246096,
-                0.03342959976196289,
-                0.032863040924072266,
-                0.03292364883422851,
-                0.03295948791503906,
-                0.03286220932006836,
-                0.03288576126098633,
-                0.03292876815795898,
-                0.03298611068725586,
-                0.033123519897460936,
-                0.032947200775146485,
-                0.03296553421020508,
-                0.03312015914916992,
-                0.033159168243408206,
-                0.03298713684082031,
-                0.03338854217529297,
-                0.032838878631591795,
-                0.03298502349853515,
-                0.03306480026245117,
-                0.03301200103759765,
-                0.03290083312988281,
-                0.032909313201904294,
-                0.032974048614501955,
-                0.03303504180908203,
-                0.032996353149414064,
-                0.032892929077148435,
-                0.0328172492980957,
-                0.03301279830932617,
-                0.03296352005004883,
-                0.03294198226928711,
-                0.03305574417114258,
-                0.03306905746459961,
-                0.033056766510009765,
-                0.03306291198730469,
-                0.03307724761962891,
-                0.033140960693359374,
-                0.033064735412597655,
-                0.03323104095458984,
-                0.032940097808837894,
-                0.03315276718139649,
-                0.03301273727416992,
-                0.03298406219482422,
-                0.03290009689331055,
-                0.03299532699584961,
-                0.03298303985595703,
-                0.03297894287109375,
-                0.03300873565673828,
-                0.032878559112548826,
-                0.03284883117675781,
-                0.03292364883422851,
-                0.032948513031005856,
-                0.032984798431396484,
-                0.033023998260498046,
-                0.03298924636840821,
-                0.03287340927124023,
-                0.032911361694335936,
-                0.03284400177001953,
-                0.03291027069091797,
-                0.033060703277587894,
-                0.033124351501464845,
-                0.03297382354736328,
-                0.03303014373779297,
-                0.03348992156982422,
-                0.03299235153198242,
-                0.033053791046142575,
-                0.03306399917602539,
-                0.032955135345458984,
-                0.032991359710693356,
-                0.032892799377441405,
-                0.0329881591796875,
-                0.03290828704833984,
-                0.03292160034179688,
-                0.03300454330444336,
-                0.033035263061523434,
-                0.033078369140625,
-                0.03300649642944336,
-                0.03344086456298828,
-                0.033134559631347656,
-                0.03300755310058594,
-                0.03307929611206055,
-                0.033031169891357424,
-                0.032952320098876955,
-                0.033089534759521484,
-                0.03298918533325195,
-                0.032939006805419925,
-                0.032966655731201173,
-                0.03299430465698242,
-                0.03299356842041016,
-                0.033051361083984376,
-                0.033113086700439456,
-                0.03297689437866211,
-                0.03301171112060547,
-                0.03291862487792969,
-                0.03299523162841797,
-                0.03288371276855469,
-                0.03292876815795898,
-                0.033075199127197266,
-                0.032998401641845705,
-                0.03321651077270508,
-                0.03300352096557617,
-                0.0330250244140625,
-                0.033081344604492184,
-                0.03307724761962891,
-                0.033104896545410156,
-                0.032911361694335936,
-                0.03300864028930664,
-                0.03287142562866211,
-                0.0329483528137207,
-                0.03305571365356445,
-                0.03292559814453125,
-                0.032968704223632815,
-                0.03297689437866211,
-                0.03288576126098633,
-                0.033040382385253905,
-                0.032901119232177735,
-                0.03300044631958008,
-                0.0329543685913086,
-                0.032833534240722655
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 30.209901795923255
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 2365.382656,
             "max_vram": 5490.343936,
             "max_reserved": 5005.901824,
             "max_allocated": 4844.946944
         },
         "latency": {
             "unit": "s",
+            "mean": 0.06139228851224749,
+            "stdev": 0.0006576332047282342,
             "values": [
+                0.06698953247070312,
+                0.06482637023925782,
+                0.061813758850097655,
+                0.061192192077636716,
+                0.06180044937133789,
+                0.06121683120727539,
+                0.061823040008544924,
+                0.061274112701416014,
+                0.061795326232910154,
+                0.06128128051757813,
+                0.06168473434448242,
+                0.0611778564453125,
+                0.061590526580810545,
+                0.06113894271850586,
+                0.06164889526367188,
+                0.06110028839111328,
+                0.06164275360107422,
+                0.061055999755859375,
+                0.06158028793334961,
+                0.06107340621948242,
+                0.06156492614746094,
+                0.06107136154174805,
+                0.06162739181518555,
+                0.061058048248291016,
+                0.06156108856201172,
+                0.06108160018920898,
+                0.06147379302978516,
+                0.060990463256835936,
+                0.061548545837402345,
+                0.061069313049316405,
+                0.06165200042724609,
+                0.061061119079589846,
+                0.061461505889892576,
+                0.060990592956542966,
+                0.06144409561157226,
+                0.06096188735961914,
+                0.06155263900756836,
+                0.06093824005126953,
+                0.06133760070800781,
+                0.060835838317871094,
+                0.06161407852172852,
+                0.060889087677001956,
+                0.061290496826171874,
+                0.060902400970458986,
+                0.06138163375854492,
+                0.06090752029418945,
+                0.06138777542114258,
+                0.06104985427856445,
+                0.06156185531616211,
+                0.06108671951293945,
+                0.06154342269897461,
+                0.06106316757202149,
+                0.061505535125732425,
+                0.061072383880615234,
+                0.0615280647277832,
+                0.061104129791259766,
+                0.06152908706665039,
+                0.06113587188720703,
+                0.061486080169677736,
+                0.060913665771484375,
+                0.06134579086303711,
+                0.06088294219970703,
+                0.06137343978881836,
+                0.060903423309326174,
+                0.06134067153930664,
+                0.06086041641235351,
+                0.06148710250854492,
+                0.061001728057861325,
+                0.06145228958129883,
+                0.06098124694824219,
+                0.06144307327270508,
+                0.0609699821472168,
+                0.061475841522216794,
+                0.06097100830078125,
+                0.061513729095458984,
+                0.060949504852294924,
+                0.06171750259399414,
+                0.06105497741699219,
+                0.06150348663330078,
+                0.06107257461547851,
+                0.06151065444946289,
+                0.061009918212890625,
+                0.061475872039794925,
+                0.06099353790283203,
+                0.06146047973632812,
+                0.06104166412353516,
+                0.06147993469238281,
+                0.06105395126342773,
+                0.06146563339233398,
+                0.061034496307373044,
+                0.06149529647827148,
+                0.060955646514892575,
+                0.0614389762878418,
+                0.061023231506347655,
+                0.06149324798583984,
+                0.061037727355957035,
+                0.06154342269897461,
+                0.06099967956542969,
+                0.06145843124389649,
+                0.06101913452148437,
+                0.061454334259033204,
+                0.06099660873413086,
+                0.0614574089050293,
+                0.060909599304199216,
+                0.06139494323730469,
+                0.060918785095214846,
+                0.06140927886962891,
+                0.061006847381591796,
+                0.06155980682373047,
+                0.06105820846557617,
+                0.06141030502319336,
+                0.0610898551940918,
+                0.0615546875,
+                0.06101606369018555,
+                0.06154751968383789,
+                0.06100409698486328,
+                0.06146480178833008,
+                0.06098944091796875,
+                0.06156390380859375,
+                0.06103551864624023,
+                0.06154240036010742,
+                0.061050880432128904,
+                0.06163046264648438,
+                0.06101212692260742,
+                0.061358081817626954,
+                0.06091263961791992,
+                0.061431808471679686,
+                0.06091263961791992,
+                0.061400062561035154,
+                0.06095974349975586,
+                0.061456382751464846,
+                0.060928256988525394,
+                0.06137036895751953,
+                0.06294323348999023,
+                0.06259609603881836,
+                0.06398259353637695,
+                0.06279183959960938,
+                0.061058048248291016,
+                0.06157619094848633,
+                0.061058048248291016,
+                0.06156185531616211,
+                0.061217025756835936,
+                0.0615464973449707,
+                0.06250393676757812,
+                0.061587455749511716,
+                0.06101401519775391,
+                0.06148940658569336,
+                0.061044734954833986,
+                0.06150758361816406,
+                0.062304256439208984,
+                0.0615464973449707,
+                0.061090816497802736,
+                0.0615333137512207,
+                0.06110723114013672,
+                0.06143916702270508,
+                0.06108671951293945,
+                0.06155596923828125,
+                0.061036544799804686,
+                0.061618175506591794,
+                0.061026302337646485,
+                0.061445121765136716,
+                0.06115238571166992,
+                0.061480960845947265
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 4169.904823615251
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 2365.382656,
             "max_vram": 5710.544896,
             "max_reserved": 5226.102784,
             "max_allocated": 4911.274496
         },
         "latency": {
             "unit": "s",
+            "mean": 10.269249523162834,
+            "stdev": 0,
             "values": [
+                10.269249523162834
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 24.83141532639109
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.04027156675750131,
+            "stdev": 0.0005416632850634589,
             "values": [
+                0.04257177734375,
+                0.04227686309814453,
+                0.040438785552978515,
+                0.04025241470336914,
+                0.04035203170776367,
+                0.040083168029785156,
+                0.040139774322509765,
+                0.04010598373413086,
+                0.040363006591796875,
+                0.040323070526123043,
+                0.04030361557006836,
+                0.04026675033569336,
+                0.040346622467041016,
+                0.0404326400756836,
+                0.04035276794433594,
+                0.040242176055908206,
+                0.04028726577758789,
+                0.04032201766967773,
+                0.04032620620727539,
+                0.04035372924804687,
+                0.040392704010009765,
+                0.04022988891601562,
+                0.04035686492919922,
+                0.04028416061401367,
+                0.04023817443847656,
+                0.04030966567993164,
+                0.040373344421386716,
+                0.0402891845703125,
+                0.040275966644287106,
+                0.040400894165039065,
+                0.04032204818725586,
+                0.040286209106445314,
+                0.0403507194519043,
+                0.040235008239746094,
+                0.04014796829223633,
+                0.04045414352416992,
+                0.0402872314453125,
+                0.04036198425292969,
+                0.040403968811035154,
+                0.04017049789428711,
+                0.04031795120239258,
+                0.040204288482666016,
+                0.04021657562255859,
+                0.040343551635742186,
+                0.040357887268066404,
+                0.040240127563476565,
+                0.04044083023071289,
+                0.040414207458496096,
+                0.04048486328125,
+                0.04031078338623047,
+                0.04032204818725586,
+                0.04019200134277344,
+                0.047661056518554686,
+                0.041534465789794923,
+                0.040325119018554685,
+                0.04011929702758789,
+                0.04045619201660156,
+                0.04031180953979492,
+                0.04044611358642578,
+                0.040282974243164064,
+                0.0403240966796875,
+                0.040237056732177735,
+                0.04026367950439453,
+                0.040395774841308595,
+                0.04018483352661133,
+                0.04016332626342774,
+                0.04030054473876953,
+                0.04032825469970703,
+                0.04040288162231445,
+                0.0405401611328125,
+                0.04029644775390625,
+                0.04018175888061523,
+                0.04032204818725586,
+                0.040323070526123043,
+                0.04020940780639649,
+                0.04011520004272461,
+                0.040156158447265625,
+                0.040427520751953126,
+                0.04040806579589844,
+                0.04041523361206055,
+                0.04009983825683594,
+                0.04023091125488281,
+                0.04048896026611328,
+                0.040199169158935545,
+                0.040272895812988284,
+                0.041350143432617184,
+                0.04156313705444336,
+                0.04027391815185547,
+                0.04037747192382812,
+                0.04008639907836914,
+                0.04024115371704102,
+                0.04010086441040039,
+                0.04025753784179688,
+                0.04028518295288086,
+                0.04046950531005859,
+                0.04053811264038086,
+                0.04035686492919922,
+                0.040392704010009765,
+                0.04051968002319336,
+                0.0401715202331543,
+                0.04017363357543945,
+                0.040140735626220704,
+                0.040374271392822264,
+                0.040325119018554685,
+                0.04017049789428711,
+                0.04024934387207031,
+                0.04025139236450195,
+                0.04024115371704102,
+                0.04040499114990234,
+                0.040269824981689455,
+                0.03996672058105469,
+                0.040025344848632814,
+                0.039972606658935546,
+                0.040158206939697266,
+                0.040308734893798825,
+                0.04019200134277344,
+                0.040226814270019534,
+                0.04015718460083008,
+                0.04025753784179688,
+                0.0402872314453125,
+                0.04021452713012695,
+                0.040153087615966795,
+                0.04032204818725586,
+                0.040360958099365234,
+                0.040258560180664066,
+                0.040275966644287106,
+                0.04021657562255859,
+                0.040275966644287106,
+                0.0401899528503418,
+                0.04012236785888672,
+                0.04008038330078125,
+                0.0401162223815918,
+                0.04002918243408203,
+                0.04017663955688477,
+                0.0400076789855957,
+                0.03996057510375976,
+                0.04003839874267578,
+                0.040018943786621096,
+                0.03996876907348633,
+                0.039907329559326174,
+                0.03992486572265625,
+                0.039940990447998044,
+                0.040256511688232424,
+                0.04002304077148437,
+                0.040150016784667966,
+                0.039989246368408206,
+                0.03999846267700195,
+                0.04000460815429688,
+                0.04017356872558594,
+                0.04035174560546875,
+                0.04016742324829101,
+                0.04012748718261719,
+                0.0403322868347168,
+                0.04016742324829101,
+                0.040048641204833986,
+                0.04006092834472656,
+                0.040240127563476565,
+                0.04009881591796875,
+                0.04012134552001953,
+                0.04022784042358398,
+                0.04029132843017578,
+                0.04015923309326172,
+                0.039992321014404295,
+                0.04015718460083008,
+                0.04031590270996094,
+                0.040174591064453126,
+                0.04025753784179688,
+                0.04016742324829101,
+                0.04012441635131836,
+                0.04002816009521484,
+                0.04011724853515625,
+                0.04014080047607422,
+                0.04044595336914063,
+                0.04051456069946289,
+                0.04025139236450195,
+                0.04036710357666016,
+                0.040182785034179686,
+                0.04026675033569336,
+                0.04014694213867188,
+                0.04022272109985352,
+                0.040197120666503904,
+                0.04013568115234375,
+                0.0399554557800293,
+                0.040089599609375,
+                0.039992321014404295,
+                0.040225791931152347,
+                0.040019966125488284,
+                0.040134654998779294,
+                0.04005068969726563,
+                0.04024627304077148,
+                0.04008755111694336,
+                0.04041625595092774,
+                0.04025753784179688,
+                0.040409088134765625,
+                0.04029747009277344,
+                0.040264705657958984,
+                0.04017663955688477,
+                0.04020940780639649,
+                0.039995391845703124,
+                0.040052734375,
+                0.04014899063110351,
+                0.040202239990234374,
+                0.04018175888061523,
+                0.040313854217529296,
+                0.04025139236450195,
+                0.04017663955688477,
+                0.04022169494628906,
+                0.04015513610839844,
+                0.04014591979980469,
+                0.04013772964477539,
+                0.04026265716552734,
+                0.04014387130737305,
+                0.040389633178710936,
+                0.040240127563476565,
+                0.04022886276245117,
+                0.04030278396606445,
+                0.03998905563354492,
+                0.04021247863769531,
+                0.04043468856811523,
+                0.040321025848388675,
+                0.040188926696777344,
+                0.04009369659423828,
+                0.0401868782043457,
+                0.04030156707763672,
+                0.04008755111694336,
+                0.04026777648925781,
+                0.04004556655883789,
+                0.04028518295288086,
+                0.04009881591796875,
+                0.04015923309326172,
+                0.04014899063110351,
+                0.04010496139526367,
+                0.04002816009521484,
+                0.039965694427490234,
+                0.04000460815429688,
+                0.039959552764892575,
+                0.04009369659423828,
+                0.04009369659423828,
+                0.04008243179321289,
+                0.03999129486083984,
+                0.04012748718261719,
+                0.03990425491333008,
+                0.039967742919921875,
+                0.040048641204833986,
+                0.03986943817138672,
+                0.039995391845703124,
+                0.040022014617919925,
+                0.039766014099121096,
+                0.03983257675170898,
+                0.04000665664672851,
+                0.04008550262451172,
+                0.039667713165283204,
+                0.040136703491210936,
+                0.04025548934936524,
+                0.04055859375
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 24.831415326391088
         },
         "energy": null,
         "efficiency": null