kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Apr 16, 2024

Commit

ec1bdb6

verified ·

1 Parent(s): 56ac8f2

Upload /gemma-7b/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

gemma-7b/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +698 -698

gemma-7b/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,202 +2,202 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1089.10592,
             "max_vram": 6534.725632,
             "max_reserved": 6312.42752,
             "max_allocated": 6191.16288
         },
         "latency": {
             "unit": "s",
-            "mean": 0.0554408369327777,
-            "stdev": 0.00046867886188081725,
             "values": [
-                0.06065068817138672,
-                0.05728492736816406,
-                0.05781708908081055,
-                0.05692211151123047,
-                0.055831550598144535,
-                0.05569740676879883,
-                0.05571993637084961,
-                0.05533695983886719,
-                0.05533900833129883,
-                0.055228416442871096,
-                0.05532057571411133,
-                0.05521612930297851,
-                0.055268383026123045,
-                0.05524172973632813,
-                0.05533494567871094,
-                0.05518745422363281,
-                0.0553639030456543,
-                0.05525708770751953,
-                0.055431167602539064,
-                0.05526323318481445,
-                0.0553853759765625,
-                0.05528268814086914,
-                0.055403518676757815,
-                0.055305217742919924,
-                0.055390209197998044,
-                0.05535232162475586,
-                0.05541580963134766,
-                0.05530316925048828,
-                0.05540454483032226,
-                0.05530112075805664,
                 0.055393280029296874,
-                0.0553454704284668,
-                0.05542521667480469,
-                0.055332862854003906,
-                0.055375873565673826,
-                0.055375873565673826,
-                0.05537279891967774,
-                0.05539123153686523,
-                0.05534515380859375,
-                0.05535232162475586,
-                0.05535027313232422,
-                0.05538611221313477,
-                0.0553809928894043,
-                0.055401470184326174,
-                0.055365631103515625,
-                0.055357662200927735,
-                0.055365631103515625,
-                0.055377983093261716,
-                0.055362560272216796,
-                0.05537811279296875,
-                0.055378944396972656,
-                0.05538304138183594,
-                0.05538508987426758,
-                0.05540249633789063,
-                0.055362560272216796,
-                0.05538611221313477,
-                0.05536767959594727,
-                0.05538304138183594,
-                0.055362560272216796,
-                0.05539123153686523,
-                0.055381183624267576,
-                0.05536483383178711,
-                0.055375873565673826,
-                0.05538105773925781,
-                0.05536675262451172,
-                0.05537996673583984,
-                0.055376895904541014,
-                0.055378944396972656,
-                0.055347198486328124,
-                0.055398399353027344,
-                0.05538304138183594,
                 0.05535539245605469,
-                0.055398399353027344,
-                0.0556124153137207,
-                0.05536972808837891,
-                0.055376895904541014,
-                0.05539865493774414,
-                0.055392257690429686,
-                0.05539430236816406,
                 0.05538918304443359,
-                0.05537279891967774,
-                0.05538412857055664,
-                0.0553779182434082,
-                0.05537996673583984,
-                0.05535334396362305,
-                0.055349246978759765,
-                0.055375873565673826,
-                0.055375873565673826,
-                0.0553779182434082,
-                0.0553963508605957,
-                0.05537513732910156,
-                0.0553779182434082,
-                0.05537295913696289,
-                0.05536153411865234,
-                0.05535232162475586,
                 0.055398399353027344,
-                0.05538816070556641,
-                0.055363582611083983,
-                0.055376895904541014,
-                0.0553994255065918,
-                0.05540454483032226,
-                0.05537388610839844,
-                0.05540761566162109,
-                0.055392257690429686,
-                0.05535129547119141,
-                0.055403518676757815,
-                0.05538304138183594,
-                0.05538611221313477,
-                0.05537279891967774,
-                0.05540761566162109,
-                0.05536870574951172,
-                0.055375873565673826,
-                0.05537177658081055,
-                0.055398399353027344,
-                0.055375873565673826,
-                0.055401470184326174,
-                0.05539430236816406,
-                0.055387134552001956,
-                0.055363582611083983,
-                0.05536460876464844,
-                0.05536665725708008,
-                0.055390209197998044,
-                0.055392257690429686,
-                0.05537411117553711,
-                0.055370750427246096,
-                0.055381088256835936,
-                0.05536972808837891,
-                0.05537897491455078,
-                0.05540454483032226,
-                0.05538918304443359,
-                0.05537177658081055,
-                0.0553809928894043,
-                0.05535641479492188,
-                0.0553809928894043,
-                0.05538508987426758,
-                0.05542911911010742,
-                0.05536153411865234,
-                0.055387134552001956,
-                0.05537279891967774,
-                0.055375873565673826,
-                0.0553779182434082,
-                0.05536767959594727,
-                0.055387134552001956,
-                0.055382015228271485,
-                0.05536870574951172,
-                0.05536460876464844,
-                0.0553779182434082,
-                0.055401470184326174,
-                0.0553809928894043,
-                0.055363582611083983,
-                0.055357440948486325,
-                0.05538508987426758,
-                0.05536460876464844,
-                0.055400447845458986,
-                0.055375873565673826,
-                0.05537279891967774,
-                0.055387134552001956,
-                0.055375873565673826,
-                0.05535948944091797,
-                0.0553809928894043,
-                0.055387134552001956,
-                0.055401470184326174,
                 0.055370750427246096,
-                0.055393310546875,
-                0.0553963508605957,
-                0.055385215759277344,
-                0.0553809928894043,
-                0.05537279891967774,
                 0.05538611221313477,
-                0.05538508987426758,
-                0.055393280029296874,
-                0.05536972808837891,
-                0.055378944396972656,
-                0.055393280029296874,
-                0.0553963508605957,
-                0.05536460876464844,
-                0.055370750427246096,
-                0.05540966415405273,
-                0.055401470184326174,
-                0.05537996673583984,
-                0.055411712646484375
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 4617.534910419937
         },
         "energy": null,
         "efficiency": null
@@ -205,23 +205,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1089.10592,
             "max_vram": 7212.105728,
             "max_reserved": 6727.663616,
             "max_allocated": 6570.75456
         },
         "latency": {
             "unit": "s",
-            "mean": 8.009259902954101,
-            "stdev": 0.0027145023345926944,
             "values": [
-                8.011974405288694,
-                8.006545400619508
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 31.838147730222477
         },
         "energy": null,
         "efficiency": null
@@ -230,524 +230,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.031408862364525884,
-            "stdev": 0.00011838795661826258,
             "values": [
-                0.0313515510559082,
-                0.03122585678100586,
-                0.03126169586181641,
-                0.031351808547973634,
-                0.03133030319213867,
-                0.03134883117675781,
-                0.03142351913452148,
-                0.031348608016967774,
-                0.031246335983276367,
-                0.0312729606628418,
-                0.0313702392578125,
-                0.031268863677978515,
-                0.03145638465881348,
-                0.03136092758178711,
-                0.0312391357421875,
-                0.03131494331359863,
-                0.03134694480895996,
-                0.03131673622131348,
-                0.03124550437927246,
-                0.03128505516052246,
-                0.031239168167114258,
-                0.03129446411132813,
-                0.03127807998657227,
-                0.031262943267822266,
-                0.031309600830078124,
-                0.03122790336608887,
-                0.03130060768127441,
-                0.031321088790893556,
-                0.03131596755981445,
-                0.03141324806213379,
-                0.031255680084228514,
-                0.031226751327514648,
-                0.03124838447570801,
-                0.03136102485656738,
-                0.031351039886474606,
-                0.031331167221069336,
-                0.03125545692443848,
-                0.031352832794189454,
-                0.03134976005554199,
-                0.03154944038391113,
-                0.03124742317199707,
-                0.03141932868957519,
-                0.03138764762878418,
-                0.03138457679748535,
-                0.031431936264038084,
-                0.031840000152587894,
-                0.03137126350402832,
-                0.03138457679748535,
-                0.0314019832611084,
-                0.031330368041992185,
-                0.0314707202911377,
-                0.03133225631713867,
-                0.03136822319030762,
-                0.031348608016967774,
-                0.031437824249267575,
-                0.031394943237304684,
-                0.031468416213989256,
-                0.03138662338256836,
-                0.031475711822509765,
-                0.0313753604888916,
-                0.031768575668334964,
-                0.03143987274169922,
-                0.03141836738586426,
-                0.0314071044921875,
-                0.03148006439208984,
-                0.031409151077270506,
-                0.031357696533203125,
-                0.03142860794067383,
-                0.03141222381591797,
-                0.031494144439697266,
-                0.03140096092224121,
-                0.03146444892883301,
-                0.031306751251220705,
-                0.03133440017700195,
-                0.03158220863342285,
-                0.03141119956970215,
-                0.03134464073181152,
-                0.03154022407531738,
-                0.03144704055786133,
-                0.03135193634033203,
-                0.031316864013671876,
-                0.03135807991027832,
-                0.031485824584960936,
-                0.03136204719543457,
-                0.03138047981262207,
-                0.03136511993408203,
-                0.03146563148498535,
-                0.031412063598632814,
-                0.03136000061035156,
-                0.031380544662475585,
-                0.03141856002807617,
-                0.0315534725189209,
-                0.031408031463623046,
-                0.03143587112426758,
-                0.03136083221435547,
-                0.03132825660705566,
-                0.03139174461364746,
-                0.03150233650207519,
-                0.031335424423217774,
-                0.03136204719543457,
-                0.03136102485656738,
-                0.031628288269042966,
-                0.03137945556640625,
-                0.03160883140563965,
-                0.03131494331359863,
-                0.03146240043640137,
-                0.03144704055786133,
-                0.031425535202026365,
-                0.03140403175354004,
-                0.031903743743896484,
-                0.03148512077331543,
-                0.031428415298461916,
-                0.031425535202026365,
-                0.031507455825805664,
-                0.031455232620239255,
-                0.031421440124511715,
-                0.03134873580932617,
-                0.031458303451538085,
-                0.031321088790893556,
-                0.0314726390838623,
-                0.03136819267272949,
-                0.03143987274169922,
-                0.03144499206542969,
-                0.03143065643310547,
-                0.0314839038848877,
-                0.03158118438720703,
-                0.03141529655456543,
-                0.03143270492553711,
-                0.031268863677978515,
-                0.03153510475158691,
-                0.03142451286315918,
-                0.03144704055786133,
-                0.031459327697753905,
-                0.031341567993164066,
-                0.031512575149536134,
-                0.031524864196777344,
-                0.03142963218688965,
-                0.031526912689208986,
-                0.03164672088623047,
-                0.03140300750732422,
-                0.03156582450866699,
-                0.03143577575683594,
-                0.031527936935424807,
-                0.03147776031494141,
-                0.031527936935424807,
-                0.03138355255126953,
-                0.031542272567749025,
-                0.031525983810424804,
-                0.031510431289672854,
-                0.03146463966369629,
-                0.031534912109375,
-                0.03142451286315918,
-                0.03149926376342774,
-                0.03144704055786133,
-                0.031437824249267575,
-                0.031440895080566404,
-                0.03150233650207519,
-                0.031456256866455076,
-                0.031442943572998046,
-                0.031441919326782225,
-                0.031488000869750975,
-                0.0314726390838623,
-                0.03146751976013184,
-                0.03146751976013184,
-                0.0314839038848877,
-                0.03147164726257324,
-                0.03157705688476563,
-                0.031460351943969726,
-                0.03143065643310547,
-                0.0314654712677002,
-                0.031491071701049804,
-                0.03152179145812988,
-                0.03146956825256347,
-                0.0318525447845459,
-                0.031525888442993165,
-                0.031456256866455076,
-                0.031512575149536134,
-                0.031488000869750975,
-                0.03146854400634766,
-                0.031508480072021484,
-                0.03150028800964356,
-                0.031496320724487305,
-                0.031504255294799804,
-                0.031512575149536134,
-                0.03148902320861816,
-                0.03155891227722168,
-                0.03148467254638672,
-                0.031527936935424807,
-                0.031513599395751955,
-                0.031666175842285156,
-                0.03143475151062012,
-                0.03140505599975586,
-                0.03135385513305664,
-                0.03137126350402832,
-                0.03134771156311035,
-                0.031438848495483396,
-                0.031486976623535154,
-                0.03129446411132813,
-                0.03139481544494629,
-                0.03137740707397461,
-                0.031348896026611325,
-                0.03142127990722656,
-                0.03130060768127441,
-                0.03139891242980957,
-                0.03129343986511231,
-                0.03141734313964844,
-                0.0314204158782959,
-                0.031401119232177734,
-                0.031381343841552736,
-                0.03150540733337402,
-                0.031422464370727536,
-                0.03136716842651367,
-                0.031356927871704104,
-                0.0314839038848877,
-                0.03143680000305176,
-                0.03139788818359375,
-                0.031453184127807614,
-                0.03139891242980957,
-                0.03149516868591309,
-                0.0314019832611084,
-                0.031409151077270506,
-                0.03141647911071777,
-                0.03126358413696289,
-                0.03133551979064941,
-                0.03134249687194824,
-                0.03156582450866699,
-                0.031291391372680666,
-                0.031441919326782225,
-                0.0313753604888916,
-                0.031352832794189454,
-                0.03127910423278808,
-                0.031457279205322264,
-                0.031460351943969726,
-                0.03149004745483398,
-                0.031339519500732424,
-                0.03143065643310547,
-                0.0315035514831543,
-                0.03153286361694336,
-                0.031440895080566404,
-                0.03132620811462403,
-                0.031308799743652346,
-                0.03137638473510742,
-                0.031351808547973634,
-                0.03137955284118652,
-                0.031256479263305666,
-                0.03151283264160156,
-                0.031357696533203125,
-                0.03136511993408203,
-                0.031357952117919925,
-                0.03146342468261719,
-                0.03138764762878418,
-                0.031421632766723634,
-                0.03139155197143555,
-                0.03177574348449707,
-                0.031253503799438476,
-                0.03136716842651367,
-                0.03134771156311035,
-                0.03122380828857422,
-                0.031291391372680666,
-                0.03122585678100586,
-                0.03141024017333984,
-                0.03135379219055176,
-                0.03135398483276367,
-                0.03129747200012207,
-                0.03135276794433594,
-                0.03103036880493164,
-                0.030929023742675782,
-                0.03122265625,
-                0.03134249687194824,
-                0.03126169586181641,
-                0.03139072036743164,
-                0.031263744354248044,
-                0.031357952117919925,
-                0.031301631927490234,
-                0.03136214447021484,
-                0.03135785675048828,
-                0.0313702392578125,
-                0.03124224090576172,
-                0.031335424423217774,
-                0.031410175323486327,
-                0.031318016052246093,
-                0.031237119674682616,
-                0.03133132743835449,
-                0.03131603240966797,
-                0.03149920082092285,
-                0.03143475151062012,
-                0.03121561622619629,
-                0.031191135406494142,
-                0.031353759765625,
-                0.03117568016052246,
-                0.03140096092224121,
-                0.031197183609008788,
-                0.03138675117492676,
-                0.03128934478759766,
-                0.03129228782653808,
-                0.03122697639465332,
-                0.03134249687194824,
-                0.031243263244628908,
-                0.031325183868408206,
-                0.031266975402832034,
-                0.031735744476318356,
-                0.031108095169067384,
-                0.031227807998657226,
-                0.031226879119873048,
-                0.03131699180603027,
-                0.031193216323852538,
-                0.03119500732421875,
-                0.03114905548095703,
-                0.031247360229492187,
-                0.031246335983276367,
-                0.03127398490905762,
-                0.031188991546630858,
-                0.031346912384033206,
-                0.03125635147094727,
-                0.031246335983276367,
-                0.03161702346801758,
-                0.031275007247924806,
-                0.031247360229492187,
-                0.031252479553222655,
-                0.03118182373046875,
-                0.031357952117919925,
-                0.031491071701049804,
-                0.031278207778930665,
-                0.03130662345886231,
-                0.03129343986511231,
-                0.031297536849975584,
-                0.031488000869750975,
-                0.03136409568786621,
-                0.03142451286315918,
-                0.031235071182250978,
-                0.03139200019836426,
-                0.03126348876953125,
-                0.031302656173706055,
-                0.03136307144165039,
-                0.031306751251220705,
-                0.03138355255126953,
-                0.03128230476379395,
-                0.031282047271728515,
-                0.03132928085327148,
-                0.03146751976013184,
-                0.03129958343505859,
-                0.03135897636413574,
-                0.03149926376342774,
-                0.031355903625488284,
-                0.031303871154785154,
-                0.03134854316711426,
-                0.0315228157043457,
-                0.031514623641967776,
-                0.031440895080566404,
-                0.03134873580932617,
-                0.031437824249267575,
-                0.03158016014099121,
-                0.03133337593078613,
-                0.031355903625488284,
-                0.03144806480407715,
-                0.03137433624267578,
-                0.03132928085327148,
-                0.03141119956970215,
-                0.031409151077270506,
-                0.03141632080078125,
-                0.0313118724822998,
-                0.03138252830505371,
-                0.03140630340576172,
-                0.0314355525970459,
-                0.031634464263916015,
-                0.031334367752075196,
-                0.03129548835754394,
-                0.03127910423278808,
-                0.03140403175354004,
-                0.031453184127807614,
-                0.031369216918945314,
-                0.031410175323486327,
-                0.031438848495483396,
-                0.031357952117919925,
-                0.03142758369445801,
-                0.031409151077270506,
-                0.03131391906738281,
-                0.03129958343505859,
-                0.03136204719543457,
-                0.03145113563537598,
-                0.03151974487304687,
-                0.031474687576293944,
-                0.03139686393737793,
-                0.0314019832611084,
-                0.03139891242980957,
-                0.03144396781921387,
-                0.03158332824707031,
-                0.03150147247314453,
-                0.031518463134765626,
-                0.031384639739990235,
-                0.03139577674865723,
-                0.031474687576293944,
-                0.03165302467346191,
-                0.031514463424682615,
-                0.03150540733337402,
-                0.031480831146240236,
-                0.0316376953125,
-                0.03143046379089356,
-                0.031491071701049804,
-                0.0315361270904541,
-                0.03150643157958984,
-                0.0318403205871582,
-                0.031548351287841794,
-                0.03155046463012695,
-                0.031410175323486327,
-                0.03141427230834961,
-                0.03153919982910156,
-                0.031388671875,
-                0.03147267150878906,
-                0.031532064437866214,
-                0.03155449676513672,
-                0.03147980880737305,
-                0.031486976623535154,
-                0.03148185539245606,
-                0.03157423973083496,
-                0.031641504287719724,
-                0.03142848014831543,
-                0.03163443183898926,
-                0.03136819267272949,
-                0.03143270492553711,
-                0.03159974479675293,
-                0.03149606323242188,
-                0.031513599395751955,
-                0.03153305625915527,
-                0.03151897621154785,
-                0.03151337623596191,
-                0.03151459121704102,
-                0.03144112014770508,
-                0.03196291160583496,
-                0.031513599395751955,
-                0.03143680000305176,
-                0.031474687576293944,
-                0.03159449577331543,
-                0.031558656692504884,
-                0.03152076721191406,
-                0.031573152542114256,
-                0.03151753616333008,
-                0.03156070327758789,
-                0.03160902404785156,
-                0.03148268890380859,
-                0.03157119941711426,
-                0.031612672805786134,
-                0.031526912689208986,
-                0.031751167297363284,
-                0.031753215789794925,
-                0.03163462448120117,
-                0.0314714241027832,
-                0.0315043830871582,
-                0.03154431915283203,
-                0.03150028800964356,
-                0.031422464370727536,
-                0.03144704055786133,
-                0.031355903625488284,
-                0.031558656692504884,
-                0.03138355255126953,
-                0.03142348861694336,
-                0.03126067161560059,
-                0.03134873580932617,
-                0.03143680000305176,
-                0.03138559913635254,
-                0.03150460815429688,
-                0.031519519805908204,
-                0.03144601631164551,
-                0.031421440124511715,
-                0.03132825660705566,
-                0.03140505599975586,
-                0.031470592498779294,
-                0.03148595237731933,
-                0.03135385513305664,
-                0.03141632080078125,
-                0.03144499206542969,
-                0.031357952117919925,
-                0.03137868881225586,
-                0.03152767944335937,
-                0.03132928085327148,
-                0.03140505599975586,
-                0.03136000061035156,
-                0.031421440124511715,
-                0.031306751251220705,
-                0.0314071044921875,
-                0.03136511993408203,
-                0.03153817558288574,
-                0.0313384952545166,
-                0.03129651260375976,
-                0.031263744354248044,
-                0.031346912384033206,
-                0.031263519287109375,
-                0.03125555229187012,
-                0.031204351425170897,
-                0.03134259223937988,
-                0.03131814384460449,
-                0.03133017539978027,
-                0.0316866569519043,
-                0.03134873580932617,
-                0.031341567993164066,
-                0.03138559913635254,
-                0.03136511993408203,
-                0.03139379119873047,
-                0.03134771156311035,
-                0.03134464073181152,
-                0.03131091117858887,
-                0.031375295639038084,
-                0.031421440124511715,
-                0.031355903625488284,
-                0.03126681518554687,
-                0.03128755187988281,
-                0.03126860809326172,
-                0.03128217506408691,
-                0.03164159965515137,
-                0.031355903625488284
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 31.838147730222477
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1087.24224,
             "max_vram": 6534.725632,
             "max_reserved": 6312.42752,
             "max_allocated": 6191.16288
         },
         "latency": {
             "unit": "s",
+            "mean": 0.055581806309315376,
+            "stdev": 0.00076348828430563,
             "values": [
+                0.06490351867675781,
+                0.05785599899291992,
+                0.0582737922668457,
+                0.05636608123779297,
+                0.05612543869018555,
+                0.05633638381958008,
+                0.056554622650146484,
+                0.05614694213867188,
+                0.05634867095947266,
+                0.05610700988769531,
+                0.055670783996582034,
+                0.05552640151977539,
                 0.055393280029296874,
+                0.05532262420654297,
+                0.05543936157226562,
+                0.055298046112060545,
+                0.05541580963134766,
+                0.05536665725708008,
+                0.05547724914550781,
                 0.05535539245605469,
+                0.05541273498535156,
                 0.05538918304443359,
+                0.055452766418457033,
                 0.055398399353027344,
+                0.05543833541870117,
+                0.055430145263671876,
+                0.05548659133911133,
                 0.055370750427246096,
+                0.055444480895996094,
+                0.05539337539672851,
+                0.05545062255859375,
                 0.05538611221313477,
+                0.05544755172729492,
+                0.05545471954345703,
+                0.05544755172729492,
+                0.0554967041015625,
+                0.05545676803588867,
+                0.05547126388549805,
+                0.05547520065307617,
+                0.05546393585205078,
+                0.05545497512817383,
+                0.05545676803588867,
+                0.055462913513183595,
+                0.055501823425292966,
+                0.05545574569702148,
+                0.05546700668334961,
+                0.05546700668334961,
+                0.05545471954345703,
+                0.05544755172729492,
+                0.05548339080810547,
+                0.055458847045898436,
+                0.05547417449951172,
+                0.055428096771240234,
+                0.05547110366821289,
+                0.05546803283691406,
+                0.055479297637939455,
+                0.05545078277587891,
+                0.055479297637939455,
+                0.05547942352294922,
+                0.05545574569702148,
+                0.05547212982177734,
+                0.05544038391113281,
+                0.05545676803588867,
+                0.0554700813293457,
+                0.055444480895996094,
+                0.05547520065307617,
+                0.05546720123291016,
+                0.05545676803588867,
+                0.05544550323486328,
+                0.05546803283691406,
+                0.05545676803588867,
+                0.05546393585205078,
+                0.055449600219726565,
+                0.05548031997680664,
+                0.05546188735961914,
+                0.05548339080810547,
+                0.05545881652832031,
+                0.05545267105102539,
+                0.0554516487121582,
+                0.05548550415039062,
+                0.05545471954345703,
+                0.05547110366821289,
+                0.055435359954833986,
+                0.055456928253173825,
+                0.05545881652832031,
+                0.05546188735961914,
+                0.05546495819091797,
+                0.05547724914550781,
+                0.05548646545410156,
+                0.055478271484375,
+                0.0554598388671875,
+                0.05545369720458984,
+                0.05544755172729492,
+                0.05547110366821289,
+                0.05546700668334961,
+                0.055482368469238284,
+                0.05549798583984375,
+                0.055479297637939455,
+                0.05545267105102539,
+                0.05543731307983398,
+                0.05544038391113281,
+                0.05547724914550781,
+                0.05543936157226562,
+                0.055465984344482425,
+                0.055431167602539064,
+                0.055462913513183595,
+                0.055419998168945314,
+                0.05547110366821289,
+                0.055444480895996094,
+                0.05549772644042969,
+                0.05552864074707031,
+                0.05546700668334961,
+                0.05546710586547852,
+                0.05546495819091797,
+                0.055462913513183595,
+                0.05547622299194336,
+                0.05547110366821289,
+                0.0555335693359375,
+                0.055488510131835936,
+                0.05580492782592773,
+                0.05545273590087891,
+                0.05545779037475586,
+                0.05544345474243164,
+                0.055521278381347655,
+                0.05551001739501953,
+                0.05551308822631836,
+                0.05549055862426758,
+                0.05548953628540039,
+                0.05546915054321289,
+                0.05548137664794922,
+                0.05546905517578125,
+                0.05549260711669922,
+                0.05547417449951172,
+                0.055512065887451174,
+                0.055465984344482425,
+                0.05554585647583008,
+                0.05546495819091797,
+                0.05549465560913086,
+                0.05548134231567383,
+                0.05551513671875,
+                0.05575065612792969,
+                0.05548543930053711,
+                0.055488510131835936,
+                0.05551103973388672,
+                0.05545062255859375,
+                0.05548646545410156,
+                0.05545779037475586,
+                0.05548543930053711,
+                0.055474433898925785,
+                0.055477470397949216,
+                0.05547622299194336,
+                0.05549260711669922,
+                0.05548134231567383,
+                0.055495681762695315,
+                0.055446529388427736,
+                0.055482368469238284,
+                0.05547417449951172,
+                0.05550899124145508,
+                0.0554967041015625,
+                0.05549158477783203,
+                0.05548953628540039,
+                0.05548748779296875,
+                0.05547212982177734,
+                0.055478271484375,
+                0.05547622299194336,
+                0.05549363327026367,
+                0.055479297637939455,
+                0.05548441696166992,
+                0.05547622299194336,
+                0.0554967041015625,
+                0.055444480895996094,
+                0.055478271484375,
+                0.055479297637939455,
+                0.05547542572021484,
+                0.05551923370361328,
+                0.0554700813293457,
+                0.05545267105102539,
+                0.05548441696166992,
+                0.05545574569702148,
+                0.055501823425292966,
+                0.05545369720458984
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 4605.8236858181235
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1087.24224,
             "max_vram": 7212.105728,
             "max_reserved": 6727.663616,
             "max_allocated": 6570.75456
         },
         "latency": {
             "unit": "s",
+            "mean": 8.930721275329589,
+            "stdev": 0.011337223052979795,
             "values": [
+                8.942058498382568,
+                8.919384052276609
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 28.553124897584404
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.035022436373841524,
+            "stdev": 0.00043531219846223377,
             "values": [
+                0.03505152130126953,
+                0.034987071990966796,
+                0.03497260665893555,
+                0.03494297790527344,
+                0.03500236892700195,
+                0.03491852951049805,
+                0.035028865814208984,
+                0.03506175994873047,
+                0.03483647918701172,
+                0.03494297790527344,
+                0.03506073760986328,
+                0.03512319946289062,
+                0.035130367279052735,
+                0.03506175994873047,
+                0.0350013427734375,
+                0.03515596771240234,
+                0.03501465606689453,
+                0.03498188781738281,
+                0.03513446426391602,
+                0.03509145736694336,
+                0.03503411102294922,
+                0.03522150421142578,
+                0.03497574234008789,
+                0.03497702407836914,
+                0.03503897476196289,
+                0.03506905746459961,
+                0.03493977737426758,
+                0.035097599029541016,
+                0.03499724960327148,
+                0.03495638275146484,
+                0.03491420745849609,
+                0.03488060760498047,
+                0.03513436889648437,
+                0.03503020858764649,
+                0.03503494262695313,
+                0.03506687927246094,
+                0.03499520111083984,
+                0.03494604873657227,
+                0.034939903259277344,
+                0.03493273544311523,
+                0.035095550537109374,
+                0.03499008178710938,
+                0.03514470291137695,
+                0.03502284622192383,
+                0.03500236892700195,
+                0.03507001495361328,
+                0.035116992950439456,
+                0.03501260757446289,
+                0.03496550369262695,
+                0.034931713104248044,
+                0.034974720001220705,
+                0.03488460922241211,
+                0.034938880920410156,
+                0.03496345520019531,
+                0.034994174957275394,
+                0.035127296447753906,
+                0.03504127883911133,
+                0.035135486602783206,
+                0.0350904312133789,
+                0.034955265045166016,
+                0.0349224967956543,
+                0.03502489471435547,
+                0.03495423889160156,
+                0.03506687927246094,
+                0.03495423889160156,
+                0.03498905563354492,
+                0.035004417419433595,
+                0.034915328979492184,
+                0.03508224105834961,
+                0.03495423889160156,
+                0.03500032043457031,
+                0.03495219039916992,
+                0.0348671989440918,
+                0.03495017623901367,
+                0.034881504058837894,
+                0.03498393630981445,
+                0.034915328979492184,
+                0.03497881698608398,
+                0.03477503967285156,
+                0.03500339126586914,
+                0.034909183502197266,
+                0.03487539291381836,
+                0.03487846374511719,
+                0.03490611267089844,
+                0.03488256072998047,
+                0.034977790832519534,
+                0.03502592086791992,
+                0.03500851058959961,
+                0.034931713104248044,
+                0.03503104019165039,
+                0.034852863311767575,
+                0.035043327331542966,
+                0.0350115852355957,
+                0.03500851058959961,
+                0.03483852767944336,
+                0.03495935821533203,
+                0.03499008178710938,
+                0.034938880920410156,
+                0.035094528198242186,
+                0.035097599029541016,
+                0.034909183502197266,
+                0.03497983932495117,
+                0.035020801544189455,
+                0.03491430282592774,
+                0.03491020965576172,
+                0.03493180847167969,
+                0.03499715042114258,
+                0.03510784149169922,
+                0.0350013427734375,
+                0.03496550369262695,
+                0.03499520111083984,
+                0.03496755218505859,
+                0.03497369766235352,
+                0.034977790832519534,
+                0.034991104125976565,
+                0.03511705780029297,
+                0.034920448303222655,
+                0.034969600677490234,
+                0.035043327331542966,
+                0.034966529846191405,
+                0.03498188781738281,
+                0.03505459213256836,
+                0.0350382080078125,
+                0.03498086547851562,
+                0.03494604873657227,
+                0.035135486602783206,
+                0.034912254333496096,
+                0.03516211318969727,
+                0.035062782287597655,
+                0.03510476684570313,
+                0.035093505859375,
+                0.035163135528564454,
+                0.034976768493652347,
+                0.03499827194213867,
+                0.03505459213256836,
+                0.03504230499267578,
+                0.03501567840576172,
+                0.03522150421142578,
+                0.035181568145751956,
+                0.03508224105834961,
+                0.03504947280883789,
+                0.034966529846191405,
+                0.03508531188964844,
+                0.03505049514770508,
+                0.034988033294677735,
+                0.034923519134521484,
+                0.03495935821533203,
+                0.03496857452392578,
+                0.03495219039916992,
+                0.03504230499267578,
+                0.035043327331542966,
+                0.03505766296386719,
+                0.03511500930786133,
+                0.03504947280883789,
+                0.035149822235107424,
+                0.03508736038208008,
+                0.035053569793701174,
+                0.03502489471435547,
+                0.03500032043457031,
+                0.03501363372802734,
+                0.03500851058959961,
+                0.03506892776489258,
+                0.03511603164672852,
+                0.03499929428100586,
+                0.035135486602783206,
+                0.03500851058959961,
+                0.035089408874511716,
+                0.035130367279052735,
+                0.03501670455932617,
+                0.035026943206787106,
+                0.035076095581054685,
+                0.03495116806030273,
+                0.035053569793701174,
+                0.03510681533813476,
+                0.03497267150878906,
+                0.03500236892700195,
+                0.034988033294677735,
+                0.034988033294677735,
+                0.03505254364013672,
+                0.035023872375488284,
+                0.03505868911743164,
+                0.03502284622192383,
+                0.034902015686035154,
+                0.034985984802246094,
+                0.03500543975830078,
+                0.03501670455932617,
+                0.035023872375488284,
+                0.03489689636230469,
+                0.034953216552734374,
+                0.03491328048706055,
+                0.03490508651733398,
+                0.03505152130126953,
+                0.03482316970825195,
+                0.034912254333496096,
+                0.03503615951538086,
+                0.03497062301635742,
+                0.03499008178710938,
+                0.034920448303222655,
+                0.034928638458251955,
+                0.03516928100585937,
+                0.0349409294128418,
+                0.035004478454589844,
+                0.03496441650390625,
+                0.03495423889160156,
+                0.03502284622192383,
+                0.03504435348510742,
+                0.034887680053710936,
+                0.03502182388305664,
+                0.03483852767944336,
+                0.03499827194213867,
+                0.03497574234008789,
+                0.0351539192199707,
+                0.0350013427734375,
+                0.035130367279052735,
+                0.035097599029541016,
+                0.03515084838867188,
+                0.034941951751708986,
+                0.0350750732421875,
+                0.03508224105834961,
+                0.03505049514770508,
+                0.03502284622192383,
+                0.03693260955810547,
+                0.03824947357177735,
+                0.039553024291992187,
+                0.03717529678344727,
+                0.03567308807373047,
+                0.0354150390625,
+                0.03525734329223633,
+                0.035119102478027346,
+                0.035269630432128905,
+                0.035318782806396484,
+                0.03530137634277344,
+                0.03519692611694336,
+                0.03510067367553711,
+                0.03502284622192383,
+                0.03520716857910156,
+                0.03528704071044922,
+                0.035143680572509765,
+                0.035122177124023435,
+                0.03504127883911133,
+                0.03506585693359375,
+                0.03507814407348633,
+                0.03493273544311523,
+                0.03501055908203125,
+                0.035004417419433595,
+                0.03496243286132812,
+                0.03496857452392578,
+                0.034917377471923826,
+                0.0349194221496582,
+                0.035059711456298825,
+                0.035105792999267575,
+                0.035020801544189455,
+                0.03492659378051758,
+                0.03497369766235352,
+                0.03503513717651367,
+                0.03499724960327148,
+                0.034872318267822264,
+                0.034748416900634765,
+                0.03489484786987305,
+                0.03474943923950195,
+                0.03486003112792969,
+                0.034928638458251955,
+                0.03489689636230469,
+                0.034769920349121096,
+                0.03482726287841797,
+                0.03479654312133789,
+                0.0348671989440918,
+                0.03494911956787109,
+                0.03494604873657227,
+                0.03483750534057617,
+                0.03498495864868164,
+                0.0348037109375,
+                0.03489996719360351,
+                0.034857982635498046,
+                0.03482316970825195,
+                0.034770942687988284,
+                0.03486521530151367,
+                0.03498899078369141,
+                0.03487558364868164,
+                0.034820926666259765,
+                0.03488972854614258,
+                0.03468185424804687,
+                0.034958335876464845,
+                0.03502899169921875,
+                0.0350300178527832,
+                0.03497574234008789,
+                0.034993152618408206,
+                0.03490611267089844,
+                0.03495219039916992,
+                0.03488358306884766,
+                0.03494400024414063,
+                0.03496345520019531,
+                0.03502592086791992,
+                0.03488870239257812,
+                0.034902015686035154,
+                0.03487744140625,
+                0.03511808013916016,
+                0.03481702423095703,
+                0.03487846374511719,
+                0.034802688598632815,
+                0.03506687927246094,
+                0.03501567840576172,
+                0.035079166412353514,
+                0.034895870208740236,
+                0.034791488647460934,
+                0.03482515335083008,
+                0.034904064178466795,
+                0.034936832427978515,
+                0.03480678558349609,
+                0.034909183502197266,
+                0.03496038436889649,
+                0.03492659378051758,
+                0.034985984802246094,
+                0.0349306869506836,
+                0.03484672164916992,
+                0.03488153457641602,
+                0.03507712173461914,
+                0.03500646209716797,
+                0.03494400024414063,
+                0.03484569549560547,
+                0.03499212646484375,
+                0.034895870208740236,
+                0.03485184097290039,
+                0.034938880920410156,
+                0.03496857452392578,
+                0.03496345520019531,
+                0.034961406707763674,
+                0.03502492904663086,
+                0.034938846588134766,
+                0.03502489471435547,
+                0.03510988616943359,
+                0.03501875305175781,
+                0.04081459045410156,
+                0.03914547348022461,
+                0.035097599029541016,
+                0.03495935821533203,
+                0.03486310577392578,
+                0.03482316970825195,
+                0.034953216552734374,
+                0.0349194221496582,
+                0.034969600677490234,
+                0.03508633422851563,
+                0.03483647918701172,
+                0.03484467315673828,
+                0.03504032135009766,
+                0.03505145645141602,
+                0.034909183502197266,
+                0.0349194221496582,
+                0.03480473709106445,
+                0.034917377471923826,
+                0.03489177703857422,
+                0.034740222930908206,
+                0.034956287384033204,
+                0.03502284622192383,
+                0.03497881698608398,
+                0.034857982635498046,
+                0.03485696029663086,
+                0.03504537582397461,
+                0.03474227142333984,
+                0.03500646209716797,
+                0.03493596649169922,
+                0.035028831481933594,
+                0.03497881698608398,
+                0.034923519134521484,
+                0.03490611267089844,
+                0.03483852767944336,
+                0.03483750534057617,
+                0.03500339126586914,
+                0.03495423889160156,
+                0.03489177703857422,
+                0.03496550369262695,
+                0.03490816116333008,
+                0.034841598510742186,
+                0.034751487731933595,
+                0.03488256072998047,
+                0.0348590087890625,
+                0.03479347229003906,
+                0.034735103607177735,
+                0.03480985641479492,
+                0.03475763320922851,
+                0.03490816116333008,
+                0.03486515045166016,
+                0.03473715209960938,
+                0.034958335876464845,
+                0.03482931137084961,
+                0.03489382553100586,
+                0.03506175994873047,
+                0.03500236892700195,
+                0.0351723518371582,
+                0.03492966461181641,
+                0.03502899169921875,
+                0.034938880920410156,
+                0.03499724960327148,
+                0.03502899169921875,
+                0.03492966461181641,
+                0.034936832427978515,
+                0.03495935821533203,
+                0.034874366760253905,
+                0.03504844665527344,
+                0.035151870727539065,
+                0.034939903259277344,
+                0.03502796936035156,
+                0.03499827194213867,
+                0.035043327331542966,
+                0.0349409294128418,
+                0.03505049514770508,
+                0.035056640625,
+                0.03490816116333008,
+                0.0350750732421875,
+                0.03493580627441406,
+                0.034925567626953126,
+                0.03503411102294922,
+                0.03502899169921875,
+                0.03525939178466797,
+                0.035007488250732424,
+                0.035171329498291014,
+                0.035160064697265625,
+                0.035124225616455076,
+                0.03506687927246094,
+                0.03497983932495117,
+                0.034958335876464845,
+                0.03502592086791992,
+                0.035194881439208986,
+                0.03497881698608398,
+                0.03514572906494141,
+                0.035060768127441404,
+                0.035085281372070315,
+                0.03511808013916016,
+                0.03515289688110351,
+                0.035095550537109374,
+                0.03506892776489258,
+                0.03501055908203125,
+                0.0350013427734375,
+                0.035020801544189455,
+                0.035064830780029296,
+                0.03501055908203125,
+                0.035062782287597655,
+                0.03500543975830078,
+                0.035076095581054685,
+                0.0350115852355957,
+                0.03504537582397461,
+                0.03506585693359375,
+                0.03505868911743164,
+                0.035007488250732424,
+                0.035156993865966796,
+                0.03500339126586914,
+                0.03506073760986328,
+                0.03489177703857422,
+                0.034966529846191405,
+                0.03495731353759766,
+                0.03489894485473633,
+                0.03504127883911133,
+                0.03490828704833984,
+                0.03490598297119141,
+                0.03493273544311523,
+                0.03496448135375976,
+                0.034941951751708986,
+                0.035043327331542966,
+                0.03484774398803711,
+                0.034994174957275394,
+                0.03484979248046875,
+                0.03488051223754883,
+                0.03491839981079101,
+                0.03488153457641602,
+                0.03482112121582031,
+                0.034700286865234374,
+                0.034974720001220705,
+                0.0350300178527832,
+                0.03492473602294922,
+                0.034847553253173826,
+                0.03484672164916992,
+                0.03494911956787109,
+                0.034863231658935546,
+                0.03501350402832031,
+                0.03488358306884766,
+                0.0348037109375,
+                0.03479040145874023,
+                0.03499520111083984,
+                0.03484569549560547,
+                0.034900993347167966,
+                0.03499724960327148,
+                0.034825214385986326,
+                0.03478015899658203,
+                0.034902015686035154,
+                0.034872318267822264,
+                0.03481087875366211,
+                0.034799617767333986,
+                0.034985984802246094,
+                0.03480985641479492,
+                0.03489484786987305,
+                0.034841598510742186,
+                0.03484262466430664,
+                0.03499520111083984,
+                0.034939903259277344,
+                0.03485184097290039,
+                0.03498188781738281,
+                0.03503411102294922,
+                0.034840576171875,
+                0.03491430282592774,
+                0.034854911804199216,
+                0.03480902481079102,
+                0.03493462371826172,
+                0.034914272308349606,
+                0.03498700714111328,
+                0.03498086547851562,
+                0.034925567626953126,
+                0.03484467315673828,
+                0.034871295928955076,
+                0.034871295928955076,
+                0.034786304473876956
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 28.5531248975844
         },
         "energy": null,
         "efficiency": null