kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Apr 16

Commit

e37953d

•

1 Parent(s): 8ff3003

Upload /Llama-2-7b-hf/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Llama-2-7b-hf/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +727 -759

Llama-2-7b-hf/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,258 +2,226 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 993.992704,
             "max_vram": 4905.238528,
             "max_reserved": 4420.796416,
             "max_allocated": 4252.667392
         },
         "latency": {
             "unit": "s",
-            "mean": 0.04236017230392006,
-            "stdev": 0.000576452064979207,
             "values": [
-                0.04680534362792969,
-                0.044624897003173826,
-                0.04397568130493164,
-                0.04385996627807617,
-                0.04318003082275391,
-                0.04338175964355469,
-                0.04339507293701172,
-                0.04385279846191406,
-                0.04769792175292969,
-                0.044948768615722653,
-                0.042513664245605466,
-                0.04241305541992187,
-                0.04255667114257813,
-                0.04243046569824219,
-                0.042534912109375,
-                0.04235776138305664,
-                0.042175487518310545,
-                0.042379264831542966,
-                0.042262527465820314,
-                0.04238950347900391,
-                0.04217036819458008,
-                0.042380287170410154,
-                0.04207513427734375,
-                0.04237209701538086,
-                0.04212736129760742,
-                0.042380287170410154,
-                0.04197683334350586,
-                0.04235468673706055,
-                0.04194815826416016,
-                0.042365951538085936,
-                0.04203519821166992,
-                0.042359809875488284,
-                0.041987071990966796,
-                0.04235059356689453,
-                0.042103809356689455,
-                0.04236800003051758,
-                0.042103809356689455,
-                0.04237516784667969,
-                0.04207206344604492,
-                0.04238131332397461,
-                0.041987071990966796,
-                0.04235161590576172,
-                0.04207001495361328,
-                0.042395648956298826,
-                0.04211404800415039,
-                0.042372222900390624,
-                0.041882625579833986,
-                0.04239769744873047,
-                0.04194918441772461,
-                0.0423741455078125,
-                0.04214374542236328,
-                0.042382335662841795,
-                0.0420136947631836,
-                0.04239974212646484,
-                0.042028030395507815,
-                0.04236083221435547,
-                0.042024959564208986,
-                0.04237004852294922,
-                0.04215500640869141,
-                0.04237619018554688,
-                0.04203212738037109,
-                0.04236800003051758,
-                0.042533889770507816,
-                0.042393600463867184,
-                0.04199935913085937,
-                0.04236800003051758,
-                0.04215398406982422,
-                0.04238643264770508,
-                0.04209868621826172,
-                0.04238249588012695,
-                0.04219107055664063,
-                0.04239769744873047,
-                0.04200960159301758,
-                0.04240387344360352,
-                0.04211097717285156,
-                0.04236800003051758,
-                0.042090496063232424,
-                0.042393600463867184,
-                0.042033153533935545,
-                0.042363903045654294,
-                0.04212838363647461,
-                0.04240691375732422,
-                0.04217446517944336,
-                0.042365951538085936,
-                0.04201062393188477,
-                0.04236697769165039,
-                0.041954303741455076,
-                0.04237004852294922,
-                0.04208025741577148,
-                0.0423741455078125,
-                0.04204544067382812,
-                0.04235776138305664,
-                0.041935871124267575,
-                0.0424007682800293,
-                0.042060798645019534,
-                0.04239379119873047,
-                0.04198809432983398,
-                0.042635265350341796,
-                0.04254617691040039,
-                0.04236288070678711,
-                0.04196454238891602,
-                0.04237311935424805,
-                0.04196454238891602,
-                0.04236083221435547,
-                0.042068992614746094,
-                0.042385406494140625,
-                0.042019840240478515,
-                0.04236185455322266,
-                0.042159103393554685,
-                0.04237721633911133,
-                0.04225024032592774,
-                0.04239257431030274,
-                0.042054656982421876,
-                0.042401790618896484,
-                0.041973758697509765,
-                0.04236185455322266,
-                0.04198310470581055,
-                0.04237823867797851,
-                0.042243072509765625,
-                0.04244377517700195,
-                0.042178848266601565,
-                0.04235468673706055,
-                0.042261505126953126,
-                0.04235161590576172,
-                0.042229759216308595,
-                0.04241113662719727,
-                0.04233011245727539,
-                0.04236697769165039,
-                0.04229836654663086,
-                0.042352638244628905,
-                0.042090496063232424,
-                0.04239257431030274,
-                0.04217036819458008,
-                0.04238979339599609,
-                0.04210892868041992,
-                0.04236697769165039,
-                0.04209664154052734,
-                0.04236288070678711,
-                0.04214988708496094,
-                0.04236185455322266,
-                0.041957374572753905,
-                0.04234649658203125,
-                0.04209868621826172,
-                0.04237209701538086,
-                0.041957374572753905,
-                0.04237004852294922,
-                0.04242124938964844,
-                0.0423741455078125,
-                0.04202598571777344,
-                0.04237004852294922,
-                0.04204032135009766,
-                0.04237721633911133,
-                0.0421212158203125,
-                0.04237107086181641,
-                0.042052608489990234,
-                0.042363903045654294,
-                0.04207820892333984,
-                0.04239974212646484,
-                0.04204467010498047,
-                0.04238848114013672,
-                0.042057727813720705,
-                0.04237721633911133,
-                0.04198092651367188,
-                0.042382431030273435,
-                0.04211814498901367,
-                0.04235161590576172,
-                0.04206489562988281,
-                0.042385406494140625,
-                0.042191871643066405,
-                0.04240691375732422,
-                0.04207820892333984,
-                0.042363903045654294,
-                0.04212838363647461,
-                0.04238848114013672,
-                0.04204339218139649,
-                0.042363903045654294,
-                0.04208844757080078,
-                0.04237007904052734,
-                0.042233089447021484,
-                0.042385406494140625,
-                0.042155105590820315,
-                0.04237516784667969,
-                0.042241024017333983,
-                0.042369022369384765,
-                0.04214169692993164,
-                0.042379264831542966,
-                0.04200755310058594,
-                0.04237516784667969,
-                0.04221644973754883,
-                0.04237004852294922,
-                0.04222054290771484,
-                0.042382335662841795,
-                0.0421396484375,
-                0.04237619018554688,
-                0.042635265350341796,
-                0.04239052963256836,
-                0.04208332824707031,
-                0.042396671295166014,
-                0.04214169692993164,
-                0.042420223236083986,
-                0.04225024032592774,
-                0.042428417205810545,
-                0.042177536010742187,
-                0.04245328140258789,
-                0.042164222717285156,
-                0.04282572937011719,
-                0.042234878540039066,
-                0.04246527862548828,
-                0.04213555145263672,
-                0.04244172668457031,
-                0.042241024017333983,
-                0.042469375610351565,
-                0.042120193481445314,
-                0.04245708847045898,
-                0.041998336791992184,
-                0.04244070434570312,
-                0.04219100952148438,
-                0.04245811080932617,
-                0.042147838592529296,
-                0.04245401763916016,
-                0.04230144119262695,
-                0.042471424102783206,
-                0.04219903945922852,
-                0.042462368011474606,
-                0.04220723342895508,
-                0.04247859191894531,
-                0.04230144119262695,
-                0.042420223236083986,
-                0.0420136947631836,
-                0.0424510383605957,
-                0.042332256317138675,
-                0.04247347259521484,
-                0.042246177673339845,
-                0.04244172668457031,
-                0.04203007888793946,
-                0.04246131134033203,
-                0.04205158233642578
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 6043.4126226703165
         },
         "energy": null,
         "efficiency": null
@@ -261,23 +229,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 993.992704,
             "max_vram": 5255.462912,
             "max_reserved": 4771.0208,
             "max_allocated": 4715.61216
         },
         "latency": {
             "unit": "s",
-            "mean": 8.584606702804567,
-            "stdev": 0.012920709609988812,
             "values": [
-                8.597527412414557,
-                8.57168599319458
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 29.704331115913813
         },
         "energy": null,
         "efficiency": null
@@ -286,524 +254,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.03366512432472383,
-            "stdev": 0.0008075856248906625,
             "values": [
-                0.03338943862915039,
-                0.03398678588867188,
-                0.03342214584350586,
-                0.033371105194091796,
-                0.03382067108154297,
-                0.03350143814086914,
-                0.03323878479003906,
-                0.03339571380615235,
-                0.033360126495361325,
-                0.0334273910522461,
-                0.03330131149291992,
-                0.033317886352539065,
-                0.033451103210449216,
-                0.033478561401367186,
-                0.03342335891723633,
-                0.03355750274658203,
-                0.033438720703125,
-                0.033372417449951175,
-                0.03343027114868164,
-                0.03374694442749023,
-                0.033312767028808594,
-                0.03350527954101563,
-                0.03364556884765625,
-                0.033463294982910154,
-                0.0334284782409668,
-                0.033459201812744144,
-                0.033420318603515624,
-                0.03340284729003906,
-                0.03374387359619141,
-                0.03347353744506836,
-                0.03338240051269531,
-                0.033462272644042966,
-                0.033318145751953125,
-                0.03343334579467774,
-                0.03338137435913086,
-                0.03347967910766601,
-                0.03346944046020508,
-                0.03356364822387695,
-                0.03350630569458008,
-                0.03352268981933594,
-                0.03346764755249024,
-                0.03343657684326172,
-                0.03349513626098633,
-                0.03339750289916992,
-                0.033438720703125,
-                0.033416191101074216,
-                0.033358848571777344,
-                0.033716224670410154,
-                0.03358745574951172,
-                0.033355552673339846,
-                0.033523681640625,
-                0.03346044921875,
-                0.03342115020751953,
-                0.0334345588684082,
-                0.03344179153442383,
-                0.033363967895507815,
-                0.033345535278320314,
-                0.03341209411621094,
-                0.0335175666809082,
-                0.033519615173339845,
-                0.03346124649047852,
-                0.03347257614135742,
-                0.0334345588684082,
-                0.033519615173339845,
-                0.03408281707763672,
-                0.03343667221069336,
-                0.033428543090820315,
-                0.033484737396240236,
-                0.03342544174194336,
-                0.03349615859985351,
-                0.03346233749389648,
-                0.03336070251464844,
-                0.0333383674621582,
-                0.033298431396484376,
-                0.033393665313720705,
-                0.033503231048583985,
-                0.0334571533203125,
-                0.03333222579956055,
-                0.033432575225830076,
-                0.03372355270385742,
-                0.033423423767089844,
-                0.03358185577392578,
-                0.033446910858154294,
-                0.033484798431396484,
-                0.03343769454956055,
-                0.03339775848388672,
-                0.033392799377441405,
-                0.03349795150756836,
-                0.03346022415161133,
-                0.03348889541625977,
-                0.03346944046020508,
-                0.03344076919555664,
-                0.033448959350585936,
-                0.033495040893554685,
-                0.033500160217285156,
-                0.03350425720214844,
-                0.033506431579589845,
-                0.03343859100341797,
-                0.033329151153564454,
-                0.033448959350585936,
-                0.033617919921875,
-                0.033460289001464846,
-                0.03367734527587891,
-                0.03338947296142578,
-                0.03341641616821289,
-                0.03336272048950195,
-                0.03341209411621094,
-                0.03331916809082031,
-                0.03346918487548828,
-                0.03339497756958008,
-                0.033419998168945315,
-                0.03339571380615235,
-                0.03351859283447266,
-                0.0334246711730957,
-                0.033495006561279295,
-                0.033521408081054686,
-                0.03341315078735352,
-                0.03335369491577148,
-                0.03343667221069336,
-                0.0332492790222168,
-                0.033269790649414065,
-                0.033390560150146485,
-                0.03363020706176758,
-                0.03348992156982422,
-                0.033484798431396484,
-                0.03378995132446289,
-                0.03370614242553711,
-                0.03338838577270508,
-                0.033539070129394534,
-                0.033501182556152344,
-                0.033432575225830076,
-                0.033377281188964845,
-                0.03328307342529297,
-                0.033364990234375,
-                0.03348787307739258,
-                0.03339775848388672,
-                0.033377281188964845,
-                0.03345305633544922,
-                0.033361183166503904,
-                0.033774303436279296,
-                0.033511425018310545,
-                0.033309696197509765,
-                0.03349734497070313,
-                0.03345280075073242,
-                0.0335206413269043,
-                0.033391616821289063,
-                0.033516544342041016,
-                0.03326464080810547,
-                0.033600513458251956,
-                0.03341107177734375,
-                0.033438720703125,
-                0.033312767028808594,
-                0.03356979370117188,
-                0.03354316711425781,
-                0.03347251129150391,
-                0.03335782241821289,
-                0.03340492630004883,
-                0.03364659118652344,
-                0.03351555252075195,
-                0.033349601745605466,
-                0.033347583770751955,
-                0.033274974822998044,
-                0.03369257736206055,
-                0.03335497665405274,
-                0.03340367889404297,
-                0.0334284782409668,
-                0.03346022415161133,
-                0.03338240051269531,
-                0.033492992401123044,
-                0.03343667221069336,
-                0.03348787307739258,
-                0.033363967895507815,
-                0.03360995101928711,
-                0.03344771194458008,
-                0.03347251129150391,
-                0.033495040893554685,
-                0.03343974304199219,
-                0.033492992401123044,
-                0.03348582458496094,
-                0.033525920867919924,
-                0.03347747039794922,
-                0.03346432113647461,
-                0.03349401473999023,
-                0.033481727600097655,
-                0.03358003234863281,
-                0.033788928985595705,
-                0.03366604614257813,
-                0.03353702545166016,
-                0.03352166366577149,
-                0.03353190231323242,
-                0.033598464965820314,
-                0.03341823959350586,
-                0.03353926467895508,
-                0.033567550659179685,
-                0.03352691268920899,
-                0.036181888580322265,
-                0.03810201644897461,
-                0.03954995346069336,
                 0.03883212661743164,
-                0.03791155242919922,
-                0.03795558547973633,
-                0.03790950393676758,
-                0.03722768020629883,
-                0.038012767791748045,
-                0.037926910400390625,
-                0.0352174072265625,
-                0.034977054595947264,
-                0.0352918701171875,
-                0.036751361846923826,
-                0.03619942474365234,
-                0.03588198471069336,
-                0.0339947509765625,
-                0.0335994873046875,
-                0.03356364822387695,
-                0.03345817565917969,
-                0.03349935913085938,
-                0.03351865768432617,
-                0.03349094390869141,
-                0.03415625762939453,
-                0.033898494720458985,
-                0.03359539031982422,
-                0.033549312591552735,
-                0.033501182556152344,
-                0.03368268966674805,
-                0.03379276657104492,
-                0.03378995132446289,
-                0.033514495849609374,
-                0.033432575225830076,
-                0.03348384094238281,
-                0.03355539321899414,
-                0.03356675338745117,
-                0.03345916748046875,
-                0.03362201690673828,
-                0.03345532989501953,
-                0.03349817657470703,
-                0.0334681282043457,
-                0.03339775848388672,
-                0.033633281707763675,
-                0.033600513458251956,
-                0.033582080841064454,
-                0.03341926574707031,
-                0.033416191101074216,
-                0.033545246124267576,
-                0.03388639831542969,
-                0.03340163040161133,
-                0.03341107177734375,
-                0.03357519912719727,
-                0.03354185485839844,
-                0.033430526733398434,
-                0.033565696716308595,
-                0.033508350372314456,
-                0.03342745590209961,
-                0.03358310317993164,
-                0.033582080841064454,
-                0.03366400146484375,
-                0.033670398712158205,
-                0.033971904754638675,
-                0.03348889541625977,
-                0.03351244735717773,
-                0.03350732803344727,
-                0.03363225555419922,
-                0.03361382293701172,
-                0.033568767547607424,
-                0.033567745208740236,
-                0.033495040893554685,
-                0.03340902328491211,
-                0.033430526733398434,
-                0.033413120269775394,
-                0.03338143920898438,
-                0.03348297500610352,
-                0.03358998489379883,
-                0.03361894226074219,
-                0.0335810546875,
-                0.033641792297363284,
-                0.03351827239990234,
-                0.033535999298095705,
-                0.033552383422851564,
-                0.03355033493041992,
-                0.03381043243408203,
-                0.03360768127441406,
-                0.033576961517333984,
-                0.03360768127441406,
-                0.033691646575927735,
-                0.033600513458251956,
-                0.03356159973144531,
-                0.03351859283447266,
-                0.03352067184448242,
-                0.033581024169921876,
-                0.03356364822387695,
-                0.03353702545166016,
-                0.03359561538696289,
-                0.033702686309814454,
-                0.03357299041748047,
-                0.033516414642333986,
-                0.033562625885009766,
-                0.03350527954101563,
-                0.03343564987182617,
-                0.034904064178466795,
-                0.040970241546630856,
-                0.03864390563964844,
-                0.03719148635864258,
-                0.03498905563354492,
-                0.03358924865722656,
-                0.03352371215820313,
-                0.03365478515625,
-                0.033597728729248044,
-                0.033610591888427736,
-                0.0341635856628418,
-                0.03346944046020508,
-                0.03350527954101563,
-                0.033552383422851564,
-                0.033416191101074216,
-                0.03345100784301758,
-                0.03351347351074219,
-                0.033614849090576174,
-                0.03357593536376953,
-                0.03394355010986328,
-                0.03347558212280274,
-                0.03351347351074219,
-                0.033519615173339845,
-                0.03357798385620117,
-                0.03370598220825195,
-                0.03351039886474609,
-                0.03357798385620117,
-                0.03352576065063476,
-                0.03349708938598633,
-                0.03343564987182617,
-                0.03348598480224609,
-                0.03346518325805664,
-                0.033685504913330076,
-                0.033486846923828126,
-                0.03360358428955078,
-                0.033529857635498046,
-                0.033616897583007815,
-                0.03362713623046875,
-                0.03355955123901367,
-                0.033549312591552735,
-                0.03355340957641602,
-                0.03346432113647461,
-                0.03357183837890625,
-                0.03360870361328125,
-                0.033805313110351565,
-                0.0337151985168457,
-                0.03364659118652344,
-                0.03357593536376953,
-                0.033800193786621094,
-                0.03346636962890625,
-                0.03344704055786133,
-                0.033489791870117185,
-                0.03355267333984375,
-                0.033667808532714845,
-                0.033565696716308595,
-                0.03355750274658203,
-                0.033560577392578124,
-                0.033492992401123044,
-                0.03346233749389648,
-                0.0334796142578125,
-                0.03374182510375977,
-                0.033655807495117186,
-                0.03349708938598633,
-                0.03366604614257813,
-                0.03388518524169922,
-                0.0335093765258789,
-                0.033519615173339845,
-                0.0337367057800293,
-                0.03350342559814453,
-                0.0334477424621582,
-                0.033532928466796875,
-                0.033503231048583985,
-                0.033568767547607424,
-                0.033540287017822266,
-                0.03367929458618164,
-                0.03362086486816406,
-                0.03359334564208984,
-                0.033565696716308595,
-                0.03343462371826172,
-                0.03383603286743164,
-                0.033587200164794925,
-                0.03348889541625977,
-                0.03375718307495117,
-                0.033414142608642575,
-                0.033588222503662106,
-                0.03347763061523437,
-                0.03347763061523437,
-                0.033337345123291014,
-                0.033503231048583985,
-                0.03336294555664063,
-                0.033465343475341795,
-                0.03362918472290039,
-                0.03353190231323242,
-                0.03345305633544922,
-                0.03349094390869141,
-                0.033516544342041016,
-                0.033514751434326175,
-                0.03344367980957031,
-                0.033514400482177735,
-                0.033461311340332034,
-                0.03348678588867188,
-                0.03369472122192383,
-                0.0335247688293457,
-                0.03361276626586914,
-                0.033328128814697267,
-                0.03353497695922852,
-                0.03355955123901367,
-                0.033859809875488284,
-                0.03348867034912109,
-                0.033459201812744144,
-                0.033452030181884765,
-                0.033474624633789064,
-                0.03349113464355469,
-                0.033398529052734376,
-                0.03338137435913086,
-                0.0337448959350586,
-                0.033459201812744144,
-                0.03349401473999023,
-                0.03330691146850586,
-                0.033372161865234375,
-                0.03338723373413086,
-                0.033410049438476565,
-                0.03351347351074219,
-                0.033576961517333984,
-                0.03346944046020508,
-                0.033807361602783206,
-                0.033500160217285156,
-                0.033498111724853515,
-                0.03347382354736328,
-                0.03347017669677734,
-                0.03367222213745117,
-                0.033472480773925783,
-                0.03373056030273437,
-                0.033329151153564454,
-                0.03344998550415039,
-                0.033372161865234375,
-                0.03342233657836914,
-                0.033447071075439455,
-                0.03367308807373047,
-                0.03379606246948242,
-                0.03354111862182617,
-                0.03330867385864258,
-                0.033406238555908206,
-                0.033344223022460935,
-                0.03347251129150391,
-                0.03360255813598633,
-                0.03360255813598633,
-                0.03338854217529297,
-                0.03337343978881836,
-                0.03332992172241211,
-                0.03338137435913086,
-                0.033442817687988284,
-                0.033516544342041016,
-                0.03348992156982422,
-                0.03335168075561523,
-                0.0334317741394043,
-                0.033481502532958986,
-                0.03336294555664063,
-                0.03345510482788086,
-                0.033649665832519535,
-                0.033530879974365234,
-                0.033484798431396484,
-                0.033414142608642575,
-                0.033345535278320314,
-                0.03343564987182617,
-                0.03360563278198242,
-                0.03374195098876953,
-                0.033631103515625,
-                0.033495040893554685,
-                0.033393665313720705,
-                0.03339775848388672,
-                0.03355033493041992,
-                0.03340390396118164,
-                0.03345305633544922,
-                0.0335175666809082,
-                0.033396961212158204,
-                0.03345078277587891,
-                0.033524734497070316,
-                0.033546398162841794,
-                0.03352479934692383,
-                0.03351631927490235,
-                0.03348992156982422,
-                0.03357491302490234,
-                0.033463294982910154,
-                0.03350630569458008,
-                0.033860607147216795,
-                0.0334284782409668,
-                0.03344815826416016,
-                0.033487648010253904,
-                0.033600513458251956,
-                0.0334951057434082,
-                0.03370284652709961,
-                0.03373382568359375,
-                0.033530689239501955,
-                0.033478656768798826,
-                0.03353702545166016,
-                0.03354111862182617,
-                0.033486846923828126,
-                0.03347455978393555,
-                0.03395481491088867,
-                0.03349094390869141,
-                0.033721343994140625,
-                0.033454078674316406,
-                0.03344076919555664,
-                0.033503231048583985,
-                0.0336363525390625,
-                0.03350630569458008,
-                0.03344384002685547,
-                0.033521854400634765,
-                0.033528640747070314,
-                0.0335206413269043,
-                0.03358924865722656,
-                0.03347267150878906
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 29.70433111591378
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 996.941824,
             "max_vram": 4905.238528,
             "max_reserved": 4420.796416,
             "max_allocated": 4252.667392
         },
         "latency": {
             "unit": "s",
+            "mean": 0.04878266125190549,
+            "stdev": 0.0009648492645063275,
             "values": [
+                0.05239081573486328,
+                0.05056512069702149,
+                0.05039820861816406,
+                0.050444286346435545,
+                0.050216960906982425,
+                0.05043404769897461,
+                0.05017292785644531,
+                0.050229248046875,
+                0.0502743034362793,
+                0.050329601287841794,
+                0.050067455291748046,
+                0.04997017669677734,
+                0.049247230529785156,
+                0.04857855987548828,
+                0.04848332977294922,
+                0.048643070220947264,
+                0.04851302337646484,
+                0.04857958221435547,
+                0.048527359008789066,
+                0.048495616912841794,
+                0.04857958221435547,
+                0.0486901741027832,
+                0.04863283157348633,
+                0.048479297637939456,
+                0.04851609420776367,
+                0.04860006332397461,
+                0.04845260620117187,
+                0.048345088958740234,
+                0.048481281280517576,
+                0.04859904098510742,
+                0.048656383514404294,
+                0.04839424133300781,
+                0.048487457275390625,
+                0.04859084701538086,
+                0.048691200256347655,
+                0.048505855560302735,
+                0.04845801544189453,
+                0.04855398559570313,
+                0.048648193359375,
+                0.048648193359375,
+                0.048415744781494144,
+                0.04839116668701172,
+                0.04853452682495117,
+                0.04859187316894531,
+                0.048601089477539064,
+                0.0484771842956543,
+                0.048677886962890625,
+                0.0485478401184082,
+                0.048395263671875,
+                0.04836761474609375,
+                0.048433151245117184,
+                0.04849971389770508,
+                0.04854272079467774,
+                0.048484352111816405,
+                0.0486379508972168,
+                0.04872806549072266,
+                0.04860211181640625,
+                0.048525310516357424,
+                0.048535743713378904,
+                0.04857855987548828,
+                0.04854988861083984,
+                0.0485399055480957,
+                0.04860416030883789,
+                0.04862156677246094,
+                0.0485928955078125,
+                0.04841676712036133,
+                0.048519168853759766,
+                0.048551937103271485,
+                0.0485181770324707,
+                0.04844748687744141,
+                0.048481281280517576,
+                0.04871094512939453,
+                0.04852838516235351,
+                0.04857241439819336,
+                0.04856422424316406,
+                0.048587776184082034,
+                0.04859494400024414,
+                0.04849356842041016,
+                0.05021286392211914,
+                0.0520079345703125,
+                0.0511723518371582,
+                0.04884275054931641,
+                0.048669696807861325,
+                0.048669696807861325,
+                0.04876083374023438,
+                0.0488355827331543,
+                0.04904345703125,
+                0.04913049697875976,
+                0.0488007698059082,
+                0.04887039947509766,
+                0.0490967025756836,
+                0.04899225616455078,
+                0.048707584381103515,
+                0.04850995254516602,
+                0.0485928955078125,
+                0.048360641479492185,
+                0.048726016998291016,
+                0.04879052734375,
+                0.04866559982299805,
+                0.04847411346435547,
+                0.0484505615234375,
+                0.048546817779541014,
+                0.04853964614868164,
+                0.04855910491943359,
+                0.04862464141845703,
+                0.04858163070678711,
+                0.04851302337646484,
+                0.04847820663452149,
+                0.04860825729370117,
+                0.04844236755371094,
+                0.04826009750366211,
+                0.04828979110717774,
+                0.048486400604248046,
+                0.04836966323852539,
+                0.048266239166259765,
+                0.048530433654785154,
+                0.048510974884033206,
+                0.048456703186035156,
+                0.04844441604614258,
+                0.048639999389648435,
+                0.04852640151977539,
+                0.04860006332397461,
+                0.04872512054443359,
+                0.04872294235229492,
+                0.048574462890625,
+                0.0485560302734375,
+                0.0486759033203125,
+                0.04873113632202149,
+                0.048699390411376955,
+                0.04873932647705078,
+                0.048492542266845705,
+                0.04858697509765625,
+                0.04853145599365234,
+                0.04860313415527344,
+                0.04854579162597656,
+                0.048753662109375,
+                0.0487823371887207,
+                0.04858572769165039,
+                0.04870988845825195,
+                0.04864921569824219,
+                0.04860019302368164,
+                0.048715774536132815,
+                0.04860518264770508,
+                0.048546817779541014,
+                0.04845260620117187,
+                0.048670783996582034,
+                0.048571392059326174,
+                0.04865536117553711,
+                0.048492542266845705,
+                0.04863078308105469,
+                0.04853452682495117,
+                0.04852633666992188,
+                0.04863385772705078,
+                0.04891033554077148,
+                0.04875059127807617,
+                0.04850713729858398,
+                0.04867686462402344,
+                0.04870655822753906,
+                0.04854687881469726,
+                0.0485928955078125,
+                0.048726016998291016,
+                0.04864102554321289,
+                0.0484771842956543,
+                0.04852121734619141,
+                0.04850096130371094,
+                0.04846899032592773,
+                0.048742401123046876,
+                0.048464897155761716,
+                0.048449535369873044,
+                0.048522239685058595,
+                0.04853452682495117,
+                0.048435199737548826,
+                0.048688129425048826,
+                0.048688129425048826,
+                0.048674816131591796,
+                0.04844134521484375,
+                0.04840038299560547,
+                0.05986115264892578,
+                0.04905990219116211,
+                0.0483768310546875,
+                0.04858367919921875,
+                0.04842086410522461,
+                0.04847001647949219,
+                0.04852019119262695,
+                0.04869222259521484,
+                0.0485294075012207,
+                0.04854297637939453,
+                0.048456703186035156,
+                0.048522239685058595,
+                0.04846899032592773,
+                0.04874137496948242,
+                0.048331775665283204,
+                0.04847820663452149,
+                0.04838399887084961,
+                0.04846899032592773,
+                0.04861337661743164,
+                0.048587776184082034,
+                0.048601089477539064,
+                0.0484505615234375,
+                0.04862771224975586,
+                0.04869734573364258,
+                0.0484403190612793,
+                0.048522239685058595,
+                0.04855910491943359,
+                0.04837376022338867
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 5247.766182292903
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 996.941824,
             "max_vram": 5255.462912,
             "max_reserved": 4771.0208,
             "max_allocated": 4715.61216
         },
         "latency": {
             "unit": "s",
+            "mean": 9.881309194564821,
+            "stdev": 0.008943601608272012,
             "values": [
+                9.87236559295655,
+                9.890252796173094
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 25.806297017834623
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.03875023213554836,
+            "stdev": 0.00012088725929289195,
             "values": [
+                0.038725631713867184,
+                0.0393891830444336,
+                0.03871750259399414,
+                0.03870304107666016,
+                0.03874303817749023,
+                0.038779903411865234,
+                0.038691841125488284,
+                0.03863654327392578,
+                0.03864371109008789,
+                0.03860172653198242,
+                0.03877478408813476,
+                0.0386324462890625,
+                0.03859763336181641,
+                0.03887308883666992,
+                0.03876454544067383,
+                0.03903180694580078,
+                0.038798336029052735,
+                0.0387665901184082,
+                0.038662143707275394,
+                0.03886899185180664,
+                0.03883724975585937,
+                0.038950912475585936,
+                0.038877185821533204,
+                0.03878937530517578,
+                0.03867315292358398,
+                0.038747135162353515,
+                0.03882393646240234,
+                0.03887308883666992,
+                0.038932479858398435,
+                0.038795265197753906,
+                0.03868057632446289,
+                0.03899596786499023,
+                0.0387061767578125,
+                0.03895500946044922,
+                0.038773761749267575,
+                0.03886899185180664,
+                0.0386693115234375,
+                0.03884236907958984,
+                0.03863347244262695,
+                0.038742015838623044,
+                0.03886796951293945,
+                0.038811649322509766,
+                0.0386693115234375,
+                0.03879935836791992,
+                0.03881267166137695,
+                0.03899903869628906,
+                0.03870435333251953,
+                0.03900188827514649,
+                0.038849536895751956,
+                0.03893350219726562,
+                0.03875328063964844,
+                0.03877068710327149,
+                0.03858943939208984,
+                0.038836223602294925,
+                0.038679550170898434,
+                0.03889871978759766,
+                0.03873276901245117,
+                0.0387696647644043,
+                0.03890790557861328,
+                0.038662143707275394,
+                0.03868057632446289,
+                0.038691841125488284,
+                0.03871027374267578,
+                0.03872256088256836,
+                0.03896627044677734,
+                0.03884246444702148,
+                0.038799263000488284,
+                0.03885977554321289,
+                0.03866739273071289,
+                0.03855859375,
+                0.038752254486083985,
+                0.03883724975585937,
+                0.0385689582824707,
+                0.03858534240722656,
+                0.03870412826538086,
+                0.03867238235473633,
+                0.03869900894165039,
+                0.038719486236572266,
+                0.038687744140625,
+                0.038785022735595705,
+                0.03878092956542969,
+                0.03868163299560547,
+                0.03868259048461914,
+                0.03876454544067383,
+                0.038604801177978515,
+                0.03883929443359375,
+                0.038577152252197267,
+                0.038687744140625,
+                0.038760448455810545,
+                0.03854848098754883,
+                0.03848396682739258,
+                0.03871846389770508,
+                0.03861324691772461,
+                0.038670078277587894,
+                0.03853209686279297,
+                0.038778881072998046,
+                0.03868979263305664,
+                0.038749183654785156,
+                0.038725631713867184,
+                0.038679550170898434,
+                0.038609920501708986,
+                0.038661121368408206,
+                0.038798336029052735,
+                0.03862015914916992,
+                0.03862835311889649,
+                0.038642688751220705,
+                0.038779903411865234,
+                0.038563838958740236,
+                0.038749183654785156,
+                0.0387665901184082,
+                0.03857408142089844,
+                0.03874508666992187,
+                0.03870032119750977,
+                0.038698719024658206,
+                0.038892833709716794,
+                0.03873455810546875,
+                0.03860172653198242,
+                0.03860889434814453,
+                0.038578174591064454,
+                0.03855769729614258,
+                0.0388218879699707,
+                0.03846041488647461,
+                0.03861708831787109,
+                0.03880243301391602,
+                0.03863859176635742,
+                0.038749183654785156,
+                0.03866624069213867,
+                0.038708225250244144,
+                0.03851059341430664,
+                0.038642688751220705,
+                0.038609920501708986,
+                0.03863040161132812,
+                0.038724609375,
+                0.03860684967041016,
+                0.03858153533935547,
+                0.03878166580200195,
+                0.03858432006835937,
+                0.03860377502441406,
+                0.03855974578857422,
+                0.03853209686279297,
+                0.03878809738159179,
+                0.038637569427490234,
+                0.03851059341430664,
+                0.03863040161132812,
+                0.038537216186523435,
+                0.038694911956787106,
+                0.03878604888916016,
+                0.03849216079711914,
+                0.03880857467651367,
+                0.038624256134033204,
+                0.03866726303100586,
+                0.038566913604736325,
+                0.03868262481689453,
+                0.03851468658447266,
+                0.038577152252197267,
+                0.038623233795166016,
+                0.03874508666992187,
+                0.03891814422607422,
+                0.03886284637451172,
+                0.038578174591064454,
+                0.03871641540527344,
+                0.038765567779541016,
+                0.03871027374267578,
+                0.03885158538818359,
+                0.03861606216430664,
+                0.038681598663330076,
+                0.038623233795166016,
+                0.03866419219970703,
+                0.038621185302734375,
+                0.038553600311279294,
+                0.03882905578613281,
+                0.038795265197753906,
+                0.039034881591796876,
+                0.03877171325683594,
+                0.03862220764160156,
+                0.0386058235168457,
+                0.038836223602294925,
+                0.038621185302734375,
+                0.0387512321472168,
+                0.038703102111816406,
+                0.03861094284057617,
+                0.03866316986083984,
+                0.038795265197753906,
+                0.03887104034423828,
+                0.038747169494628905,
+                0.03864163208007813,
+                0.038778881072998046,
+                0.03893657684326172,
+                0.038675457000732424,
+                0.03865702438354492,
+                0.03872153472900391,
+                0.03865292739868164,
+                0.03878809738159179,
+                0.038844417572021485,
+                0.038626304626464845,
+                0.03876249694824219,
+                0.03875020980834961,
+                0.03853955078125,
+                0.03871612930297851,
+                0.038781951904296875,
+                0.038781951904296875,
+                0.03870003128051758,
+                0.03855769729614258,
+                0.038773761749267575,
+                0.03889561462402344,
+                0.03879731369018555,
+                0.038523902893066404,
+                0.03865599822998047,
+                0.0385054702758789,
+                0.03887308883666992,
+                0.038504447937011715,
+                0.038580223083496096,
+                0.038640640258789063,
+                0.038645759582519534,
+                0.03863040161132812,
+                0.03874816131591797,
+                0.038553600311279294,
+                0.03871334457397461,
+                0.03841126251220703,
+                0.038752254486083985,
+                0.03862732696533203,
+                0.03864473724365235,
+                0.03871027374267578,
+                0.03880038452148438,
+                0.038942718505859376,
+                0.038983680725097655,
+                0.03877478408813476,
+                0.038470657348632815,
+                0.038604801177978515,
+                0.038787071228027346,
+                0.038523902893066404,
+                0.03877478408813476,
+                0.03856076812744141,
+                0.03871744155883789,
+                0.03854438400268555,
+                0.03879731369018555,
+                0.038763519287109374,
+                0.03882905578613281,
+                0.03871846389770508,
+                0.038563838958740236,
+                0.038577152252197267,
+                0.03873791885375977,
+                0.03882086563110351,
+                0.03860172653198242,
+                0.038703102111816406,
+                0.038768638610839845,
+                0.03865292739868164,
+                0.03882495880126953,
+                0.03885260772705078,
+                0.03864883041381836,
+                0.03866419219970703,
+                0.03870105743408203,
+                0.03851980972290039,
+                0.038763519287109374,
+                0.038919166564941404,
+                0.03899289703369141,
+                0.03885977554321289,
+                0.03891712188720703,
+                0.03871641540527344,
+                0.038675457000732424,
+                0.03880448150634765,
+                0.03876454544067383,
+                0.0387327995300293,
+                0.03883020782470703,
+                0.038819713592529295,
+                0.0388935661315918,
+                0.03884236907958984,
+                0.03879935836791992,
+                0.03885055923461914,
+                0.038811649322509766,
+                0.03882495880126953,
+                0.03899596786499023,
+                0.03901440048217773,
+                0.03888844680786133,
+                0.03875532913208008,
+                0.03873791885375977,
+                0.03878400039672852,
+                0.03889766311645508,
+                0.03887104034423828,
+                0.03877283096313477,
+                0.03875727844238281,
+                0.0387583999633789,
+                0.038747135162353515,
+                0.03889766311645508,
+                0.03880550384521484,
+                0.03891712188720703,
+                0.03887820816040039,
+                0.03874303817749023,
+                0.03868364715576172,
+                0.038618240356445316,
+                0.03872345733642578,
+                0.03888435363769531,
+                0.03870441436767578,
+                0.03878575897216797,
+                0.03867750549316406,
+                0.03876761627197266,
+                0.038708225250244144,
+                0.03872358322143555,
+                0.03880755233764648,
+                0.03874303817749023,
+                0.03877171325683594,
+                0.0386324462890625,
+                0.03871027374267578,
+                0.03868057632446289,
+                0.03865804672241211,
+                0.03858534240722656,
+                0.03864678573608398,
+                0.03867036819458008,
+                0.038844383239746094,
+                0.0388218879699707,
+                0.03902361679077149,
+                0.03868364715576172,
+                0.038763519287109374,
+                0.038773761749267575,
+                0.03884134292602539,
+                0.03891404724121094,
+                0.038803680419921875,
+                0.03879913711547851,
+                0.03878092956542969,
+                0.03880243301391602,
+                0.03880448150634765,
+                0.03877785491943359,
+                0.03880652618408203,
+                0.03864998245239258,
+                0.03891084671020508,
+                0.03881881713867188,
+                0.03874611282348633,
+                0.038836223602294925,
+                0.03877785491943359,
+                0.03878809738159179,
+                0.03905228805541992,
+                0.038776832580566405,
+                0.039003135681152344,
+                0.03871744155883789,
+                0.038886398315429685,
+                0.03873484802246094,
+                0.03877785491943359,
+                0.03871641540527344,
+                0.03892428970336914,
+                0.038773761749267575,
+                0.03872256088256836,
+                0.038693984985351565,
+                0.03877571105957031,
+                0.038860801696777345,
+                0.03878092956542969,
+                0.038747135162353515,
+                0.038537216186523435,
+                0.038730751037597655,
+                0.038640640258789063,
+                0.03882393646240234,
+                0.03879443359375,
+                0.038677310943603514,
+                0.03847884750366211,
+                0.038730751037597655,
+                0.038798336029052735,
+                0.038957054138183594,
+                0.0387665901184082,
+                0.03885260772705078,
+                0.03869388961791992,
+                0.03884646224975586,
+                0.03879423904418945,
+                0.03907174301147461,
+                0.03901440048217773,
+                0.03895296096801758,
+                0.03896627044677734,
+                0.038795486450195316,
+                0.038776607513427735,
+                0.038798336029052735,
+                0.03880755233764648,
+                0.038919166564941404,
+                0.03883520126342774,
+                0.03883724975585937,
+                0.03890790557861328,
+                0.03877273559570313,
+                0.038575103759765625,
+                0.03882905578613281,
+                0.038904830932617186,
+                0.03887206268310547,
+                0.03895603179931641,
+                0.0387665901184082,
+                0.03871027374267578,
+                0.03874303817749023,
+                0.0388218879699707,
+                0.03893062210083008,
+                0.03865068817138672,
+                0.038801406860351564,
+                0.03868672180175781,
+                0.038670337677001954,
+                0.03861606216430664,
+                0.03868467330932617,
+                0.038725631713867184,
+                0.03870943832397461,
+                0.038922046661376955,
+                0.03868979263305664,
+                0.03868262481689453,
+                0.03861811065673828,
+                0.03866944122314453,
+                0.038738815307617185,
+                0.038645759582519534,
+                0.03879423904418945,
+                0.03875430297851563,
+                0.03866828918457031,
+                0.0385904655456543,
+                0.03874611282348633,
+                0.03873382568359375,
+                0.03876147079467773,
+                0.038566913604736325,
+                0.038609920501708986,
+                0.03877273559570313,
+                0.03888947296142578,
+                0.03875635147094727,
+                0.038727680206298826,
+                0.038626304626464845,
+                0.03880550384521484,
+                0.0388935661315918,
+                0.03891302490234375,
+                0.038919166564941404,
+                0.038740993499755856,
+                0.03880857467651367,
+                0.038749183654785156,
+                0.038972415924072266,
+                0.03890380859375,
+                0.038659072875976565,
+                0.03876147079467773,
+                0.03876249694824219,
+                0.03880448150634765,
+                0.03875020980834961,
+                0.038844417572021485,
+                0.03885465621948242,
+                0.03905535888671875,
+                0.03890687942504883,
+                0.038744064331054685,
+                0.038575103759765625,
+                0.03894988632202148,
+                0.038687744140625,
+                0.038787071228027346,
+                0.038742015838623044,
+                0.03877171325683594,
+                0.038730751037597655,
+                0.03863449478149414,
+                0.038703102111816406,
+                0.03894476699829102,
+                0.03861094284057617,
+                0.03891507339477539,
+                0.038787071228027346,
+                0.03899084854125977,
+                0.038607872009277344,
+                0.03865599822998047,
+                0.038778881072998046,
+                0.038830078125,
+                0.03886899185180664,
+                0.03865615844726562,
+                0.03884323120117188,
+                0.03894579315185547,
+                0.03885670471191406,
+                0.03890585708618164,
+                0.038811649322509766,
+                0.03862015914916992,
+                0.03888844680786133,
+                0.03883929443359375,
+                0.0388485107421875,
+                0.03871539306640625,
+                0.03881369781494141,
+                0.038744064331054685,
+                0.038863872528076174,
+                0.03887820816040039,
+                0.03881369781494141,
+                0.03862527847290039,
                 0.03883212661743164,
+                0.03882393646240234,
+                0.03887615966796875,
+                0.03879731369018555,
+                0.03880755233764648,
+                0.03863654327392578,
+                0.038727680206298826,
+                0.038747135162353515,
+                0.03871539306640625,
+                0.03880243301391602,
+                0.03880652618408203,
+                0.038752254486083985,
+                0.038623233795166016,
+                0.03860070419311523,
+                0.038712318420410154,
+                0.038724609375,
+                0.038553600311279294,
+                0.038860801696777345,
+                0.03886182403564453,
+                0.03885260772705078,
+                0.039008255004882815,
+                0.03885158538818359,
+                0.03877478408813476,
+                0.038757377624511716,
+                0.03877171325683594,
+                0.03884339141845703,
+                0.03890687942504883,
+                0.03890995025634766,
+                0.038849536895751956,
+                0.038724609375,
+                0.03881062316894531,
+                0.03874816131591797,
+                0.03883827209472656,
+                0.0387583999633789,
+                0.03891507339477539,
+                0.038795265197753906,
+                0.03872358322143555,
+                0.03869081497192383,
+                0.038719486236572266,
+                0.03872051239013672,
+                0.03887820816040039,
+                0.03859763336181641
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 25.8062970178346
         },
         "energy": null,
         "efficiency": null