kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Mar 14, 2024

Commit

466a138

•

1 Parent(s): 45d3807

Upload /Llama-2-7b-hf/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Llama-2-7b-hf/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +731 -732

Llama-2-7b-hf/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,258 +2,257 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1014.673408,
             "max_vram": 4905.238528,
             "max_reserved": 4420.796416,
             "max_allocated": 4252.667392
         },
         "latency": {
             "unit": "s",
-            "mean": 0.04235845992333779,
-            "stdev": 0.000575210706458403,
             "values": [
-                0.04604630279541016,
-                0.04459417724609375,
-                0.044017696380615236,
-                0.043,
-                0.04298649597167969,
-                0.044181758880615235,
-                0.04794060897827149,
-                0.045223072052001954,
-                0.04260659027099609,
-                0.04237311935424805,
-                0.04183859252929688,
-                0.04231884765625,
-                0.04192665481567383,
-                0.04237004852294922,
-                0.04188876724243164,
-                0.04235776138305664,
-                0.042006526947021484,
-                0.04234137725830078,
-                0.04183567810058594,
-                0.04234649658203125,
-                0.04187033462524414,
-                0.04235878372192383,
-                0.04184985733032227,
-                0.04235878372192383,
-                0.041902080535888675,
-                0.04236083221435547,
-                0.04187340927124023,
-                0.04234137725830078,
-                0.041842689514160154,
-                0.04272127914428711,
-                0.041869312286376956,
                 0.04237823867797851,
-                0.041842689514160154,
-                0.042352638244628905,
-                0.041847808837890625,
-                0.042379264831542966,
-                0.04184783935546875,
-                0.04233830261230469,
-                0.041853950500488284,
-                0.04236083221435547,
-                0.04184371185302734,
-                0.042344703674316406,
-                0.04214886474609375,
-                0.04237331390380859,
-                0.04184985733032227,
-                0.04237516784667969,
-                0.0418682861328125,
-                0.04235878372192383,
-                0.041821182250976564,
-                0.04236697769165039,
-                0.041850879669189454,
-                0.04236288070678711,
-                0.04184371185302734,
-                0.04235059356689453,
-                0.04187363052368164,
-                0.042355712890625,
-                0.042249214172363284,
                 0.04237619018554688,
-                0.04187955093383789,
                 0.04239257431030274,
-                0.0418785285949707,
-                0.04235366439819336,
-                0.04186531066894531,
-                0.042355712890625,
-                0.041917537689208986,
-                0.04249190521240234,
-                0.04222566223144531,
                 0.042352638244628905,
-                0.04187855911254883,
-                0.04234649658203125,
-                0.04184064102172851,
-                0.042349567413330076,
-                0.04185497665405274,
-                0.04238438415527344,
-                0.04185190582275391,
-                0.04234985733032227,
-                0.042038272857666016,
-                0.04237107086181641,
-                0.042256385803222656,
-                0.042438751220703126,
-                0.04217958450317383,
-                0.0424089584350586,
-                0.04188671875,
-                0.04234137725830078,
-                0.04185804748535156,
-                0.04234137725830078,
-                0.04192256164550781,
-                0.04234649658203125,
-                0.04183254241943359,
-                0.04236492919921875,
-                0.04184985733032227,
-                0.0423454704284668,
-                0.04186111831665039,
-                0.042359809875488284,
-                0.04212758255004883,
-                0.04239974212646484,
-                0.04209766387939453,
-                0.04239769744873047,
-                0.0418600959777832,
-                0.042365951538085936,
-                0.04186111831665039,
-                0.042379264831542966,
-                0.041869312286376956,
-                0.04236492919921875,
-                0.04219190216064453,
-                0.042369022369384765,
-                0.04218572616577149,
                 0.042369022369384765,
-                0.04184473419189453,
-                0.04236492919921875,
-                0.04185702514648437,
-                0.04235366439819336,
-                0.041853950500488284,
-                0.04245708847045898,
-                0.04187750244140625,
                 0.042382335662841795,
-                0.0421662712097168,
-                0.04239257431030274,
-                0.04235161590576172,
-                0.04247552108764648,
-                0.04256563186645508,
-                0.04246220779418945,
-                0.04232601547241211,
-                0.04243046569824219,
-                0.04306022262573242,
                 0.042395648956298826,
-                0.042397758483886716,
-                0.042433536529541016,
-                0.04243558502197266,
-                0.042396961212158205,
-                0.042479713439941405,
-                0.04241305541992187,
-                0.04252262496948242,
                 0.042379264831542966,
-                0.042336254119873046,
-                0.042444801330566405,
-                0.04249497604370117,
-                0.0424161262512207,
-                0.04234649658203125,
-                0.04237823867797851,
-                0.04191129684448242,
-                0.04237516784667969,
-                0.04225024032592774,
-                0.0424192008972168,
-                0.04230656051635742,
-                0.04242329788208008,
                 0.04240588760375977,
-                0.04239180755615234,
-                0.04234672164916992,
                 0.042385406494140625,
-                0.04242432022094727,
-                0.042499073028564455,
-                0.04243558502197266,
-                0.0424192008972168,
-                0.042444801330566405,
-                0.04238950347900391,
-                0.04229529571533203,
-                0.042387454986572266,
-                0.042387454986572266,
-                0.04235488128662109,
-                0.042378303527832034,
-                0.04238950347900391,
-                0.04216115188598633,
-                0.04242432022094727,
-                0.042482688903808595,
-                0.04240793609619141,
-                0.04228915023803711,
-                0.04238643264770508,
-                0.04234137725830078,
-                0.04237516784667969,
                 0.04225228881835937,
-                0.04238643264770508,
-                0.04278988647460937,
                 0.04238336181640625,
-                0.04233523178100586,
-                0.042380287170410154,
-                0.04213555145263672,
-                0.0424051513671875,
-                0.04213452911376953,
-                0.042353759765625,
-                0.041837825775146484,
                 0.042349567413330076,
-                0.0418785285949707,
-                0.04242329788208008,
-                0.04229017639160156,
-                0.04263628768920898,
-                0.042299678802490234,
-                0.04239974212646484,
-                0.042503360748291016,
-                0.04242943954467773,
-                0.042280960083007815,
-                0.04239974212646484,
                 0.042363903045654294,
-                0.042439678192138675,
                 0.042352638244628905,
-                0.042404895782470704,
-                0.04229836654663086,
-                0.04256972885131836,
-                0.04243046569824219,
-                0.042403839111328126,
-                0.04239769744873047,
                 0.04239257431030274,
-                0.04236083221435547,
-                0.04242432022094727,
-                0.04230553436279297,
-                0.042396671295166014,
-                0.04221644973754883,
-                0.042379264831542966,
-                0.04259430313110352,
-                0.04235776138305664,
-                0.04236800003051758,
-                0.04237311935424805,
-                0.042365951538085936,
-                0.04242227172851563,
-                0.042560672760009764,
                 0.042385406494140625,
-                0.04235776138305664,
-                0.04237619018554688,
-                0.04247347259521484,
-                0.04280934524536133,
-                0.042359809875488284,
-                0.04239769744873047,
-                0.04246323013305664,
-                0.042395648956298826,
-                0.042229759216308595,
-                0.04241305541992187,
-                0.042379264831542966,
-                0.042379264831542966,
-                0.04227686309814453,
-                0.04240486526489258,
-                0.04219190216064453,
-                0.04236288070678711,
-                0.0423004150390625,
-                0.042365951538085936,
-                0.042259456634521485,
-                0.04236697769165039,
-                0.042383392333984374
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 6043.656933309665
         },
         "energy": null,
         "efficiency": null
@@ -261,23 +260,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1014.673408,
             "max_vram": 5255.462912,
             "max_reserved": 4771.0208,
-            "max_allocated": 4715.611648
         },
         "latency": {
             "unit": "s",
-            "mean": 8.496251308441167,
-            "stdev": 0.006892139434813593,
             "values": [
-                8.489359169006352,
-                8.50314344787598
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 30.01323651369084
         },
         "energy": null,
         "efficiency": null
@@ -286,524 +285,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.033318632582122205,
-            "stdev": 0.0001715019741849696,
             "values": [
-                0.03346720123291016,
-                0.03380940628051758,
-                0.033414142608642575,
-                0.03335782241821289,
-                0.03330867385864258,
-                0.033291263580322264,
-                0.03315609741210938,
-                0.0333199348449707,
-                0.03316940689086914,
-                0.0333383674621582,
-                0.0332861442565918,
-                0.03326566314697266,
-                0.03335168075561523,
-                0.03326873779296875,
-                0.033446910858154294,
-                0.033567745208740236,
-                0.033219585418701174,
-                0.033255424499511715,
-                0.033175552368164066,
-                0.03319193649291992,
-                0.03323392105102539,
-                0.03323801422119141,
                 0.033708030700683594,
-                0.0332861442565918,
-                0.03327897644042969,
-                0.033274879455566404,
-                0.03344384002685547,
-                0.033271808624267575,
-                0.033372161865234375,
-                0.033377281188964845,
-                0.03337420654296875,
-                0.033328128814697267,
-                0.03321753692626953,
-                0.03337011337280273,
-                0.033259521484375,
-                0.03322470474243164,
-                0.03324825668334961,
                 0.03370086288452148,
-                0.033255424499511715,
-                0.03335782241821289,
-                0.033307647705078124,
-                0.033285118103027346,
-                0.03321446228027344,
-                0.033290241241455076,
-                0.033334270477294925,
-                0.033416191101074216,
-                0.03401830291748047,
-                0.03400908660888672,
-                0.03389132690429687,
-                0.03330559921264648,
-                0.03326976013183594,
-                0.03334368133544922,
-                0.03332281494140625,
-                0.03337932968139649,
-                0.033296382904052735,
-                0.035366912841796876,
-                0.03342335891723633,
-                0.033235969543457033,
-                0.03321139144897461,
-                0.03332198333740234,
-                0.03336294555664063,
-                0.03337625503540039,
-                0.033347583770751955,
-                0.033261566162109374,
-                0.033377281188964845,
-                0.03352678298950195,
-                0.03329536056518555,
-                0.033296382904052735,
-                0.033291263580322264,
-                0.03328409576416016,
-                0.03331174468994141,
-                0.033258495330810545,
-                0.033219585418701174,
-                0.033219585418701174,
-                0.03324415969848633,
-                0.03327590560913086,
-                0.033263614654541016,
-                0.033285118103027346,
-                0.03314483261108398,
-                0.03323494338989258,
-                0.0332861442565918,
-                0.0332492790222168,
-                0.03318067169189453,
-                0.033701889038085936,
-                0.03316121673583984,
-                0.0332492790222168,
-                0.03317452621459961,
-                0.033314815521240236,
-                0.03318783950805664,
-                0.033288192749023435,
-                0.03322265625,
-                0.03326873779296875,
-                0.03327590560913086,
-                0.03323187255859375,
-                0.03310182571411133,
-                0.033255424499511715,
-                0.033142784118652346,
                 0.033650688171386715,
-                0.03315097427368164,
-                0.033307647705078124,
-                0.03320012664794922,
-                0.03325235366821289,
-                0.03338240051269531,
-                0.033293312072753906,
-                0.03321753692626953,
-                0.03321465682983398,
-                0.03318764877319336,
-                0.03331686401367188,
-                0.03322367858886719,
-                0.03321855926513672,
-                0.0332042236328125,
-                0.03322675323486328,
-                0.03317657470703125,
-                0.0332677116394043,
-                0.033380352020263675,
-                0.033414142608642575,
-                0.03347455978393555,
-                0.03338137435913086,
-                0.03322265625,
-                0.033468414306640625,
-                0.0333568000793457,
-                0.03338444900512695,
-                0.03336703872680664,
-                0.03322470474243164,
-                0.033470462799072266,
-                0.03316121673583984,
-                0.033160190582275394,
-                0.033225727081298825,
-                0.03322675323486328,
-                0.03319014358520508,
-                0.033279743194580075,
-                0.03321855926513672,
-                0.03325337600708008,
-                0.03334656143188477,
-                0.03335782241821289,
-                0.03322470474243164,
-                0.03336908721923828,
-                0.03313356781005859,
-                0.033172481536865236,
-                0.03320012664794922,
-                0.0332410888671875,
-                0.03342335891723633,
-                0.03331891250610351,
-                0.033825790405273434,
-                0.03311820983886719,
-                0.03343360137939453,
-                0.033291263580322264,
-                0.03317279815673828,
-                0.03330835342407226,
-                0.03316428756713867,
-                0.03321241760253906,
-                0.03336601638793945,
-                0.033107967376708985,
-                0.03324313735961914,
-                0.033320960998535154,
-                0.033124351501464845,
-                0.03309056091308594,
-                0.033544193267822264,
-                0.033555454254150394,
-                0.03325747299194336,
-                0.03314585494995117,
-                0.033423583984375,
-                0.03320195388793945,
-                0.03326668930053711,
-                0.033263614654541016,
-                0.03348582458496094,
-                0.03338444900512695,
-                0.033186817169189455,
-                0.03367628860473633,
-                0.03368038558959961,
-                0.03340083312988281,
-                0.033301502227783206,
-                0.03327283096313476,
-                0.0331038703918457,
-                0.0331141128540039,
-                0.033296382904052735,
-                0.03316223907470703,
-                0.03312639999389649,
-                0.033113086700439456,
-                0.03316428756713867,
-                0.03325030517578125,
-                0.03342233657836914,
-                0.03314585494995117,
-                0.0331673583984375,
-                0.03301273727416992,
-                0.03329232025146484,
-                0.0331734733581543,
-                0.03319705581665039,
-                0.03306496047973633,
-                0.033276927947998046,
-                0.033083393096923826,
-                0.033170433044433595,
-                0.03320832061767578,
-                0.03324518585205078,
-                0.03321548843383789,
-                0.03318272018432617,
-                0.03312844848632813,
-                0.0331960334777832,
-                0.03331686401367188,
-                0.03302195358276367,
-                0.03327078247070313,
-                0.033020927429199216,
-                0.03313971328735352,
-                0.03311001586914063,
                 0.03360665512084961,
-                0.033329151153564454,
-                0.033143806457519534,
-                0.03317782211303711,
-                0.033045280456542966,
-                0.03308544158935547,
-                0.0331141128540039,
-                0.03321241760253906,
-                0.03304959869384766,
-                0.033111038208007815,
-                0.03309465789794922,
-                0.03321241760253906,
-                0.03324415969848633,
-                0.03318486404418945,
-                0.03342633438110352,
-                0.033380352020263675,
-                0.03314688110351562,
-                0.033235969543457033,
-                0.03321343994140625,
-                0.033331199645996096,
-                0.03318272018432617,
-                0.03340595245361328,
-                0.03321855926513672,
-                0.033263614654541016,
-                0.03313049697875976,
-                0.03328409576416016,
-                0.03324006271362305,
-                0.03340697479248047,
-                0.033242111206054685,
-                0.03329536056518555,
-                0.03313663864135742,
-                0.03324518585205078,
-                0.03329536056518555,
-                0.033291263580322264,
-                0.03312844848632813,
-                0.03336191940307617,
-                0.03318067169189453,
-                0.03320832061767578,
-                0.033323009490966796,
-                0.03341107177734375,
-                0.03322880172729492,
-                0.03324518585205078,
-                0.033230846405029296,
-                0.03338137435913086,
-                0.0330618896484375,
-                0.03305574417114258,
-                0.033113086700439456,
-                0.033364990234375,
-                0.03319807815551758,
-                0.03337420654296875,
-                0.03324825668334961,
-                0.033560577392578124,
-                0.0333383674621582,
-                0.03336806488037109,
-                0.033272064208984375,
-                0.03331865692138672,
-                0.03336703872680664,
-                0.03326876831054688,
-                0.03331683349609375,
-                0.03364863967895508,
-                0.03325439834594727,
-                0.033377281188964845,
-                0.033259521484375,
-                0.03315609741210938,
-                0.03319091033935547,
-                0.03331174468994141,
-                0.033337345123291014,
-                0.0330885124206543,
-                0.03322880172729492,
-                0.03315814590454102,
-                0.033154048919677735,
-                0.03321036911010742,
-                0.033137664794921876,
-                0.03323801422119141,
-                0.03346636962890625,
-                0.033058815002441407,
-                0.03313971328735352,
-                0.03320627212524414,
-                0.0331776008605957,
-                0.03320115280151367,
-                0.03321651077270508,
-                0.033285118103027346,
-                0.03320217514038086,
-                0.03319398498535156,
-                0.03333324813842774,
-                0.03325030517578125,
-                0.03329945755004883,
-                0.03313868713378906,
-                0.033148929595947264,
-                0.033159168243408206,
-                0.03326464080810547,
-                0.033230846405029296,
-                0.03321446228027344,
-                0.033276927947998046,
-                0.033344512939453126,
-                0.03339775848388672,
-                0.0332564468383789,
-                0.033276927947998046,
-                0.033326175689697264,
-                0.03324918365478516,
-                0.0333834228515625,
-                0.03355340957641602,
-                0.03336806488037109,
-                0.03344793701171875,
-                0.03344384002685547,
-                0.033290241241455076,
-                0.033430526733398434,
-                0.0331253776550293,
-                0.03326367950439453,
-                0.033176513671875,
-                0.033307647705078124,
-                0.03341516876220703,
-                0.03333017730712891,
-                0.033277950286865234,
-                0.0333568000793457,
-                0.03335372924804687,
-                0.033455265045166015,
-                0.033401695251464844,
-                0.033288192749023435,
-                0.033329151153564454,
-                0.033827838897705076,
-                0.033544193267822264,
                 0.033500160217285156,
-                0.033445888519287106,
                 0.03355648040771484,
-                0.03351859283447266,
-                0.033858558654785154,
-                0.03339468765258789,
-                0.03349094390869141,
-                0.03340185546875,
-                0.033432575225830076,
-                0.033538047790527346,
-                0.033470462799072266,
-                0.03341926574707031,
-                0.03385651016235352,
-                0.03338649749755859,
-                0.0335093765258789,
-                0.03346432113647461,
-                0.03339263916015625,
                 0.03343564987182617,
-                0.03352166366577149,
-                0.033414142608642575,
-                0.03345100784301758,
-                0.033337345123291014,
-                0.033454078674316406,
-                0.03328716659545899,
-                0.033503231048583985,
-                0.03328307342529297,
-                0.03344076919555664,
-                0.033410049438476565,
-                0.03337932968139649,
                 0.0335810546875,
-                0.03328409576416016,
                 0.03346636962890625,
-                0.03332819366455078,
-                0.03340486526489258,
-                0.03334883117675781,
-                0.03342006301879883,
-                0.033514495849609374,
-                0.03355750274658203,
                 0.03342745590209961,
-                0.03348787307739258,
-                0.03352576065063476,
-                0.03352268981933594,
-                0.033358016967773435,
-                0.033468223571777346,
-                0.03339775848388672,
-                0.033430526733398434,
-                0.03353190231323242,
-                0.033342464447021485,
-                0.03337113571166992,
-                0.033320960998535154,
                 0.03354009628295898,
-                0.03343667221069336,
-                0.03338751983642578,
-                0.03338956832885742,
-                0.033463294982910154,
-                0.0332861442565918,
-                0.03337932968139649,
-                0.03340390396118164,
-                0.03373875045776367,
-                0.03335475158691406,
-                0.03340595245361328,
-                0.03340083312988281,
-                0.03331584167480469,
-                0.03341209411621094,
-                0.03326873779296875,
-                0.033306625366210936,
-                0.033342464447021485,
-                0.03333529663085937,
-                0.033393665313720705,
-                0.03337113571166992,
-                0.03336908721923828,
-                0.03347558212280274,
-                0.03350732803344727,
-                0.0336445426940918,
-                0.03340902328491211,
-                0.03340288162231445,
-                0.03340185546875,
-                0.03337420654296875,
-                0.03329740905761719,
-                0.03334348678588867,
-                0.033375232696533204,
-                0.033413120269775394,
-                0.03320832061767578,
-                0.03330252838134766,
-                0.03333855819702149,
-                0.033346366882324216,
-                0.03326566314697266,
-                0.03337113571166992,
-                0.03346944046020508,
-                0.033393665313720705,
-                0.033498111724853515,
-                0.0334284782409668,
-                0.03336191940307617,
-                0.033323009490966796,
-                0.03340697479248047,
-                0.03341110229492188,
-                0.0332789421081543,
-                0.03335372924804687,
-                0.03322060775756836,
-                0.03331071853637695,
-                0.033242271423339846,
-                0.03330134582519531,
-                0.033205249786376956,
-                0.03320115280151367,
-                0.03326566314697266,
-                0.03329536056518555,
-                0.033342464447021485,
-                0.03318476867675781,
-                0.0333496322631836,
-                0.03325337600708008,
-                0.033328128814697267,
-                0.03326566314697266,
-                0.033203201293945314,
-                0.033329151153564454,
-                0.033285118103027346,
-                0.03334041595458984,
-                0.033342464447021485,
-                0.03336191940307617,
-                0.033380352020263675,
-                0.03370086288452148,
-                0.03334764862060547,
-                0.033365951538085935,
-                0.03345305633544922,
-                0.033372161865234375,
-                0.03334860610961914,
-                0.0332861442565918,
-                0.03339468765258789,
-                0.03320115280151367,
-                0.033328128814697267,
-                0.03332710266113281,
-                0.03342335891723633,
-                0.033372161865234375,
-                0.033271808624267575,
                 0.03356979370117188,
-                0.03353497695922852,
-                0.03361894226074219,
-                0.033181697845458984,
-                0.033328128814697267,
-                0.03331174468994141,
-                0.03318476867675781,
-                0.03319500732421875,
-                0.033261566162109374,
-                0.033519615173339845,
-                0.03323392105102539,
-                0.033285118103027346,
-                0.03306905746459961,
-                0.03324313735961914,
-                0.033328128814697267,
-                0.033113086700439456,
-                0.033312767028808594,
-                0.0332492790222168,
-                0.03352371215820313,
-                0.03318179321289062,
-                0.03322051239013672,
-                0.03315302276611328,
-                0.03323699188232422,
-                0.033152000427246094,
-                0.03322777557373047,
-                0.03332419204711914,
-                0.03319075012207031,
-                0.03314688110351562,
-                0.03334348678588867,
-                0.03321343994140625,
-                0.033181697845458984,
-                0.03316857528686523,
-                0.03333100891113281,
-                0.03328921508789062,
-                0.03334041595458984,
-                0.03347763061523437,
-                0.03329740905761719,
-                0.03316633605957031,
-                0.03334656143188477,
-                0.03329945755004883,
-                0.033334270477294925,
-                0.033111038208007815,
-                0.03321855926513672,
-                0.033140735626220705,
-                0.03334656143188477,
-                0.03337324905395508,
-                0.03357382583618164,
-                0.033154048919677735,
-                0.03328204727172852,
-                0.033148929595947264,
-                0.033307647705078124
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 30.013236513690856
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1015.881728,
             "max_vram": 4905.238528,
             "max_reserved": 4420.796416,
             "max_allocated": 4252.667392
         },
         "latency": {
             "unit": "s",
+            "mean": 0.042467162730330105,
+            "stdev": 0.0006609647717465549,
             "values": [
+                0.048517601013183594,
+                0.04331315231323242,
+                0.043079681396484375,
+                0.04321279907226563,
+                0.04323123168945313,
+                0.043218944549560545,
+                0.043480064392089846,
+                0.04548710250854492,
+                0.04911718368530273,
+                0.0445296630859375,
+                0.042159103393554685,
                 0.04237823867797851,
+                0.04234444808959961,
+                0.0423741455078125,
+                0.04237619018554688,
+                0.04238336181640625,
+                0.04228633499145508,
+                0.042385406494140625,
+                0.04228409576416016,
+                0.04238336181640625,
+                0.04239462280273437,
+                0.04246220779418945,
+                0.042264575958251956,
+                0.042359809875488284,
+                0.04222873687744141,
+                0.042570751190185545,
+                0.04218265533447266,
+                0.042363903045654294,
+                0.04226764678955078,
+                0.04237209701538086,
+                0.04212940979003906,
+                0.042363903045654294,
+                0.04219612884521484,
+                0.04238156890869141,
+                0.04231782531738281,
+                0.04241408157348633,
+                0.042324031829833984,
+                0.042387454986572266,
+                0.04229529571533203,
+                0.04235776138305664,
+                0.04212838363647461,
+                0.04236800003051758,
+                0.04221440124511719,
+                0.04236800003051758,
+                0.04280115127563477,
                 0.04237619018554688,
+                0.04229430389404297,
+                0.0424007682800293,
+                0.04217139053344727,
+                0.04236185455322266,
+                0.04226867294311523,
+                0.042393600463867184,
+                0.042243072509765625,
                 0.04239257431030274,
+                0.04221952056884765,
                 0.042352638244628905,
+                0.042827934265136716,
+                0.04238643264770508,
                 0.042369022369384765,
+                0.0424161262512207,
+                0.04222873687744141,
                 0.042382335662841795,
+                0.042278911590576174,
+                0.04239052963256836,
+                0.042237953186035154,
+                0.04238147354125977,
+                0.042308609008789064,
+                0.04239052963256836,
+                0.04216934585571289,
+                0.042385406494140625,
+                0.04228710556030273,
+                0.0423741455078125,
+                0.04220927810668945,
+                0.04236185455322266,
+                0.04229119873046875,
+                0.04249103927612305,
                 0.042395648956298826,
+                0.04238131332397461,
+                0.04235673522949219,
+                0.04242124938964844,
+                0.04230144119262695,
+                0.04241116714477539,
+                0.042425342559814457,
+                0.04239052963256836,
+                0.042248191833496096,
+                0.04243865585327149,
+                0.04233126449584961,
+                0.04239769744873047,
+                0.04232191848754883,
+                0.042420223236083986,
+                0.042331134796142575,
                 0.042379264831542966,
+                0.0426506233215332,
+                0.04239462280273437,
+                0.04236697769165039,
+                0.042393600463867184,
+                0.042396671295166014,
+                0.04249292755126953,
+                0.04233011245727539,
+                0.042396671295166014,
+                0.04233011245727539,
+                0.042482688903808595,
+                0.042241024017333983,
+                0.04264243316650391,
+                0.04236288070678711,
+                0.042363903045654294,
+                0.04232191848754883,
+                0.04237311935424805,
+                0.042431488037109374,
+                0.04239052963256836,
+                0.0423741455078125,
+                0.042412033081054686,
+                0.04233523178100586,
+                0.04239462280273437,
+                0.042395648956298826,
+                0.04238950347900391,
+                0.04253696060180664,
+                0.04238131332397461,
+                0.042426368713378904,
+                0.04243865585327149,
+                0.042415103912353515,
+                0.042521598815917966,
+                0.042327041625976565,
                 0.04240588760375977,
+                0.042412033081054686,
+                0.04235059356689453,
+                0.042264575958251956,
+                0.042379264831542966,
+                0.04227174377441406,
+                0.042433761596679685,
+                0.04225331115722656,
+                0.04241222381591797,
+                0.04238131332397461,
+                0.04242943954467773,
+                0.04240691375732422,
+                0.042363903045654294,
+                0.04223004913330078,
                 0.042385406494140625,
+                0.04230758285522461,
+                0.04237004852294922,
+                0.042744831085205076,
+                0.04236697769165039,
                 0.04225228881835937,
+                0.04240793609619141,
+                0.04233830261230469,
                 0.04238336181640625,
+                0.042308609008789064,
+                0.042395648956298826,
+                0.04227993774414063,
+                0.04239567947387695,
+                0.04232499313354492,
+                0.042812416076660156,
+                0.0422083511352539,
+                0.04241100692749023,
+                0.042234878540039066,
+                0.042392894744873046,
+                0.04212940979003906,
+                0.04237823867797851,
+                0.04230451202392578,
+                0.042379264831542966,
+                0.04227174377441406,
+                0.04237619018554688,
+                0.04218982315063476,
+                0.04238131332397461,
+                0.04234854507446289,
+                0.04237516784667969,
+                0.04220927810668945,
+                0.042409984588623044,
                 0.042349567413330076,
+                0.04237004852294922,
+                0.04219903945922852,
+                0.04237516784667969,
+                0.04221440124511719,
                 0.042363903045654294,
+                0.04225228881835937,
+                0.042369022369384765,
+                0.04212428665161133,
+                0.04236800003051758,
+                0.0422737922668457,
+                0.04237107086181641,
+                0.04219903945922852,
+                0.0424161262512207,
+                0.04229529571533203,
+                0.04235366439819336,
+                0.042248191833496096,
+                0.04236800003051758,
+                0.04218572616577149,
+                0.04277248001098633,
+                0.042382335662841795,
+                0.042355712890625,
+                0.042428417205810545,
+                0.04238643264770508,
                 0.042352638244628905,
+                0.04240281677246094,
+                0.04240281677246094,
+                0.042387454986572266,
+                0.042262527465820314,
+                0.04239974212646484,
+                0.04257484817504883,
+                0.042591552734375,
+                0.04235673522949219,
+                0.042393600463867184,
+                0.04225024032592774,
+                0.04242345428466797,
+                0.04231782531738281,
+                0.04240691375732422,
+                0.04241222381591797,
+                0.04261171340942383,
+                0.042246337890625,
+                0.04240793609619141,
+                0.04248883056640625,
+                0.042380287170410154,
+                0.04228505706787109,
+                0.04241100692749023,
+                0.042417152404785156,
+                0.04240691375732422,
                 0.04239257431030274,
+                0.042386497497558594,
+                0.04246441650390625,
+                0.04236185455322266,
+                0.042369022369384765,
+                0.042417152404785156,
+                0.042482688903808595,
                 0.042385406494140625,
+                0.042297344207763675,
+                0.04238848114013672,
+                0.04245811080932617,
+                0.04240281677246094,
+                0.04224227142333985,
+                0.04238336181640625,
+                0.042420223236083986,
+                0.042389823913574216,
+                0.04247465515136719,
+                0.042393600463867184,
+                0.042292224884033204,
+                0.0428042221069336
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 6028.187040081311
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1015.881728,
             "max_vram": 5255.462912,
             "max_reserved": 4771.0208,
+            "max_allocated": 4715.61216
         },
         "latency": {
             "unit": "s",
+            "mean": 8.599395771026611,
+            "stdev": 0.03203231048584154,
             "values": [
+                8.56736346054077,
+                8.631428081512453
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 29.653246203547816
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.033723120670692594,
+            "stdev": 0.0007542579259829491,
             "values": [
+                0.03367903900146484,
+                0.034095104217529294,
+                0.03362924957275391,
+                0.033646526336669924,
+                0.033672191619873046,
+                0.03379916763305664,
+                0.03361587142944336,
+                0.03376639938354492,
+                0.03357798385620117,
+                0.03353190231323242,
+                0.033588222503662106,
+                0.033863937377929684,
+                0.03352755355834961,
+                0.033552383422851564,
+                0.03365494537353515,
+                0.033521503448486326,
+                0.03344486236572266,
+                0.03351980972290039,
+                0.0335296630859375,
+                0.03351244735717773,
+                0.033667072296142575,
+                0.03366400146484375,
+                0.03353702545166016,
+                0.03360665512084961,
+                0.033732608795166014,
+                0.033527809143066405,
+                0.033498111724853515,
+                0.033559776306152346,
+                0.033648414611816405,
+                0.03360153579711914,
+                0.03352678298950195,
+                0.03374694442749023,
+                0.0335781135559082,
+                0.03363315200805664,
+                0.033604606628417966,
+                0.03352371215820313,
+                0.033670143127441404,
+                0.03367424011230469,
+                0.03356979370117188,
+                0.03373056030273437,
                 0.033708030700683594,
+                0.03353395080566406,
+                0.033614849090576174,
+                0.03355852890014648,
+                0.03376332855224609,
+                0.03358617782592774,
+                0.033514495849609374,
+                0.033640735626220705,
+                0.033441600799560545,
+                0.03353900909423828,
+                0.03348188781738281,
+                0.03361465454101562,
+                0.03352268981933594,
+                0.033567745208740236,
+                0.03363759994506836,
+                0.033532703399658206,
+                0.0335175666809082,
+                0.0339835205078125,
+                0.03363119888305664,
+                0.0335728645324707,
+                0.033532928466796875,
+                0.03351347351074219,
+                0.03356979370117188,
+                0.03359231948852539,
+                0.03363225555419922,
+                0.03357807922363281,
+                0.03343247985839844,
+                0.03353497695922852,
+                0.03351347351074219,
+                0.0334202880859375,
+                0.03354009628295898,
+                0.033796127319335935,
+                0.03371942520141601,
+                0.03353379058837891,
+                0.03362918472290039,
+                0.03350140762329101,
+                0.03364742279052734,
+                0.03346121597290039,
+                0.0335022087097168,
+                0.0335175666809082,
+                0.033584129333496096,
+                0.033710079193115236,
+                0.03355574417114258,
+                0.03373641586303711,
+                0.03346944046020508,
+                0.033544193267822264,
+                0.03350352096557617,
+                0.03355516815185547,
+                0.033481727600097655,
+                0.03347359848022461,
+                0.0335082893371582,
+                0.033446910858154294,
+                0.03348992156982422,
+                0.03352883148193359,
+                0.033433727264404293,
+                0.03351744079589844,
+                0.0336445426940918,
+                0.03349708938598633,
+                0.03350527954101563,
+                0.033516544342041016,
+                0.033527935028076175,
+                0.03341606521606445,
+                0.03358003234863281,
+                0.03350527954101563,
+                0.03360563278198242,
+                0.0335093765258789,
+                0.03366604614257813,
+                0.03351574325561523,
+                0.03360847854614258,
+                0.03349913787841797,
+                0.033751041412353515,
+                0.03354214477539062,
                 0.03370086288452148,
+                0.03360153579711914,
+                0.033567745208740236,
+                0.03361280059814453,
+                0.03362508773803711,
+                0.03391795349121094,
+                0.03364863967895508,
+                0.033498111724853515,
+                0.033689697265625,
+                0.03361782455444336,
+                0.03367424011230469,
+                0.03368243026733399,
+                0.033576961517333984,
+                0.033584129333496096,
+                0.0337520637512207,
+                0.03362303924560547,
+                0.03363532638549805,
+                0.03349094390869141,
+                0.03344412612915039,
+                0.03370774459838867,
+                0.03375833511352539,
+                0.03355532836914062,
+                0.03360358428955078,
+                0.0337336311340332,
+                0.03346124649047852,
+                0.033522720336914065,
+                0.033508319854736325,
+                0.03347353744506836,
+                0.033693695068359376,
+                0.033535999298095705,
+                0.03356467056274414,
+                0.03382271957397461,
+                0.03361382293701172,
+                0.033459201812744144,
+                0.03358310317993164,
+                0.03362303924560547,
+                0.0335032958984375,
+                0.03348678588867188,
+                0.03362937545776367,
+                0.033528640747070314,
+                0.03379916763305664,
                 0.033650688171386715,
+                0.033576961517333984,
+                0.03346739196777344,
+                0.033620990753173825,
+                0.033701889038085936,
+                0.033667072296142575,
+                0.033514495849609374,
+                0.03357593536376953,
+                0.033729534149169925,
+                0.033511425018310545,
+                0.03358617782592774,
+                0.03355033493041992,
+                0.033583263397216796,
+                0.033570655822753905,
+                0.03360358428955078,
+                0.03356364822387695,
                 0.03360665512084961,
+                0.03374899291992187,
+                0.033718273162841796,
+                0.03359743881225586,
+                0.033584129333496096,
+                0.03356364822387695,
+                0.0336629753112793,
+                0.033620990753173825,
+                0.033919998168945316,
+                0.03372768020629883,
+                0.033554271697998045,
+                0.03358832168579102,
+                0.03361881637573242,
+                0.03367833709716797,
+                0.03359027099609375,
+                0.03363532638549805,
+                0.03375001525878906,
+                0.03360873413085937,
+                0.033424510955810546,
+                0.03377056121826172,
+                0.03363510513305664,
+                0.033546241760253906,
+                0.03377356719970703,
+                0.033805313110351565,
+                0.03364147186279297,
+                0.033600608825683595,
+                0.03353180694580078,
+                0.0335912971496582,
+                0.03360768127441406,
+                0.03358415985107422,
+                0.03361276626586914,
+                0.03357900619506836,
+                0.03366912078857422,
+                0.03369267272949219,
+                0.033539070129394534,
+                0.033519615173339845,
+                0.033642494201660156,
+                0.03354019165039063,
+                0.033525665283203124,
+                0.03361711883544922,
+                0.03361974334716797,
+                0.03349724960327148,
+                0.033764190673828125,
+                0.0334837760925293,
+                0.033565696716308595,
+                0.03359027099609375,
+                0.033501182556152344,
+                0.033549312591552735,
+                0.03354111862182617,
+                0.03348787307739258,
+                0.033649665832519535,
+                0.03352371215820313,
+                0.03361075210571289,
+                0.03359539031982422,
+                0.033685504913330076,
+                0.03357491302490234,
+                0.03379097747802735,
+                0.03350630569458008,
+                0.033620990753173825,
+                0.03346944046020508,
+                0.033530048370361325,
+                0.03350713729858398,
+                0.0335022087097168,
+                0.03343974304199219,
+                0.0335912971496582,
+                0.03343462371826172,
+                0.03355955123901367,
+                0.03342131042480469,
+                0.0340316162109375,
+                0.033539070129394534,
+                0.03346944046020508,
+                0.03349606323242187,
+                0.03350425720214844,
                 0.033500160217285156,
+                0.03361280059814453,
+                0.033909759521484374,
+                0.033649665832519535,
+                0.03361996841430664,
+                0.03359231948852539,
+                0.033546241760253906,
+                0.0335206413269043,
+                0.0335994873046875,
+                0.033827838897705076,
+                0.03380326461791992,
+                0.03357900619506836,
+                0.033546432495117184,
+                0.033893375396728515,
+                0.033667072296142575,
+                0.033667072296142575,
+                0.03363942337036133,
+                0.033555454254150394,
+                0.033598464965820314,
+                0.033509407043457035,
+                0.03363529586791992,
+                0.033582080841064454,
+                0.03367424011230469,
+                0.03362303924560547,
+                0.03373875045776367,
+                0.033576961517333984,
+                0.033649761199951174,
+                0.03349187088012695,
+                0.03364863967895508,
+                0.033584320068359375,
+                0.033551166534423825,
+                0.03348787307739258,
+                0.033719551086425784,
+                0.03360128021240234,
+                0.03377151870727539,
+                0.03352492904663086,
+                0.03367200088500977,
+                0.03359231948852539,
+                0.03360255813598633,
                 0.03355648040771484,
+                0.03363532638549805,
+                0.033584129333496096,
+                0.033532928466796875,
+                0.03352268981933594,
+                0.033827838897705076,
+                0.03350630569458008,
+                0.0335728645324707,
+                0.033484798431396484,
+                0.03364556884765625,
+                0.033462272644042966,
+                0.03363225555419922,
+                0.033565696716308595,
+                0.033704959869384765,
+                0.03361894226074219,
+                0.034223102569580076,
+                0.03355340957641602,
+                0.03362835311889648,
+                0.03361260986328125,
+                0.033585151672363284,
+                0.03353811264038086,
+                0.033615806579589846,
                 0.03343564987182617,
+                0.03364044952392578,
+                0.0336363525390625,
+                0.033729534149169925,
+                0.03359539031982422,
+                0.033632320404052736,
+                0.03352467346191406,
+                0.03403673553466797,
+                0.03348889541625977,
+                0.03359334564208984,
+                0.033519615173339845,
+                0.03355257415771484,
+                0.033492801666259765,
+                0.03364371109008789,
+                0.033594303131103516,
+                0.03369868850708008,
+                0.03351859283447266,
+                0.03353190231323242,
+                0.0335206413269043,
+                0.033588222503662106,
+                0.03347763061523437,
+                0.03348992156982422,
+                0.03362815856933594,
+                0.03365692901611328,
+                0.03376630401611328,
+                0.03358003234863281,
+                0.03367628860473633,
+                0.033670143127441404,
+                0.033584129333496096,
+                0.03367628860473633,
+                0.033584129333496096,
+                0.03358003234863281,
+                0.03368960189819336,
+                0.03364863967895508,
+                0.0335206413269043,
+                0.033588222503662106,
+                0.033568767547607424,
+                0.03355459213256836,
+                0.033698654174804686,
+                0.03367424011230469,
+                0.03360563278198242,
+                0.033555454254150394,
+                0.03350735855102539,
+                0.033648609161376956,
+                0.03350732803344727,
+                0.03362918472290039,
+                0.033598464965820314,
+                0.03358310317993164,
+                0.033928192138671875,
+                0.03349923324584961,
+                0.03360348892211914,
+                0.0335728645324707,
+                0.03362201690673828,
+                0.03391795349121094,
+                0.033555454254150394,
+                0.03356159973144531,
+                0.0336814079284668,
+                0.033651744842529294,
+                0.03361497497558594,
+                0.0335450553894043,
+                0.03351580810546875,
+                0.03373846435546875,
                 0.0335810546875,
+                0.033465343475341795,
+                0.03353702545166016,
+                0.0335013427734375,
+                0.03357782363891602,
+                0.03403571319580078,
+                0.033627166748046874,
+                0.03364767837524414,
+                0.03356047821044922,
+                0.033729534149169925,
+                0.03368054580688477,
+                0.033608543395996095,
+                0.033582080841064454,
+                0.03363532638549805,
+                0.03355955123901367,
+                0.03358310317993164,
+                0.03362508773803711,
+                0.03377766418457031,
+                0.03361820983886719,
+                0.033477344512939454,
+                0.033567745208740236,
+                0.03341033554077148,
+                0.03353673553466797,
+                0.03342131042480469,
+                0.0334837760925293,
                 0.03346636962890625,
+                0.03382601547241211,
+                0.033547039031982424,
+                0.033486881256103516,
+                0.03350012969970703,
+                0.03364147186279297,
+                0.03338751983642578,
+                0.033544193267822264,
+                0.03377356719970703,
+                0.03358617782592774,
+                0.03361996841430664,
                 0.03342745590209961,
+                0.03360358428955078,
+                0.03361587142944336,
+                0.0336732177734375,
+                0.03346432113647461,
+                0.033535999298095705,
+                0.033462272644042966,
+                0.03345817565917969,
+                0.033616897583007815,
+                0.033699840545654294,
+                0.03346124649047852,
                 0.03354009628295898,
+                0.033508350372314456,
+                0.03351248168945312,
+                0.033476638793945315,
+                0.03420975875854492,
+                0.03355542373657226,
+                0.0336097297668457,
+                0.03361177444458008,
+                0.03355033493041992,
+                0.033569950103759766,
+                0.03987760162353516,
+                0.039078048706054684,
+                0.03925167846679688,
+                0.0383559684753418,
+                0.038001953125,
+                0.03823791885375977,
+                0.03890585708618164,
+                0.03825151824951172,
+                0.03803955078125,
+                0.03559936141967773,
+                0.03567308807373047,
+                0.03725107192993164,
+                0.03808870315551758,
+                0.036789249420166016,
+                0.03564748764038086,
+                0.03366937637329102,
+                0.03364134216308594,
+                0.03357171249389648,
+                0.033396736145019534,
+                0.03352883148193359,
+                0.033667072296142575,
+                0.03363020706176758,
+                0.033617919921875,
+                0.033544193267822264,
+                0.033683456420898435,
                 0.03356979370117188,
+                0.03340508651733398,
+                0.03358294296264648,
+                0.033942527770996093,
+                0.03343769454956055,
+                0.03358924865722656,
+                0.03358323287963867,
+                0.03440729522705078,
+                0.03371212768554688,
+                0.03364761734008789,
+                0.03349932861328125,
+                0.033621822357177734,
+                0.033546241760253906,
+                0.03354217529296875,
+                0.03358307266235352,
+                0.03364863967895508,
+                0.03382089614868164,
+                0.0336146240234375,
+                0.03368960189819336,
+                0.03351039886474609,
+                0.03358937454223633,
+                0.03356659317016602,
+                0.03346124649047852,
+                0.03352166366577149,
+                0.03363532638549805,
+                0.03348787307739258,
+                0.033537311553955076,
+                0.03388079833984375,
+                0.033529857635498046,
+                0.033588222503662106,
+                0.03347251129150391,
+                0.03355875015258789,
+                0.03351836776733398,
+                0.03363020706176758,
+                0.033732769012451175,
+                0.03356143951416016,
+                0.03349094390869141,
+                0.03346022415161133,
+                0.03352390289306641,
+                0.033569599151611326,
+                0.033637374877929685,
+                0.034272254943847655,
+                0.0335810546875,
+                0.03355648040771484,
+                0.033538047790527346,
+                0.03346636962890625,
+                0.03358924865722656,
+                0.03346636962890625,
+                0.033432575225830076,
+                0.033492992401123044,
+                0.033538238525390625,
+                0.03342623901367187,
+                0.03347251129150391,
+                0.033470462799072266,
+                0.03345510482788086,
+                0.033585151672363284,
+                0.03352166366577149,
+                0.03354745483398437,
+                0.03356550216674805,
+                0.033538047790527346,
+                0.033567745208740236,
+                0.03365683364868164,
+                0.03360665512084961
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 29.653246203547816
         },
         "energy": null,
         "efficiency": null