kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Mar 14, 2024

Commit

e6c8cd9

verified ·

1 Parent(s): 23e8419

Upload /Llama-2-7b-hf-AWQ/AWQ_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Llama-2-7b-hf-AWQ/AWQ_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +310 -802

Llama-2-7b-hf-AWQ/AWQ_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,294 +2,58 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 965.369856,
-            "max_vram": 2585.788416,
-            "max_reserved": 2101.346304,
-            "max_allocated": 2014.912
         },
         "latency": {
             "unit": "s",
-            "mean": 0.03665172828422797,
-            "stdev": 0.0004557043892313135,
             "values": [
-                0.03982793426513672,
-                0.03812579345703125,
-                0.03785420989990235,
-                0.03777024078369141,
-                0.03776716613769531,
-                0.03766988754272461,
-                0.037795841217041014,
-                0.03792588806152344,
-                0.037718017578125,
-                0.041485313415527345,
-                0.039076862335205076,
-                0.03761459350585938,
-                0.037032958984375,
-                0.03664486312866211,
-                0.03660726547241211,
-                0.03688972854614258,
-                0.03658956909179688,
-                0.0365926399230957,
-                0.036588577270507815,
-                0.03654246520996094,
-                0.036536319732666016,
-                0.03654655838012695,
-                0.03664384078979492,
-                0.03650790405273437,
-                0.036557823181152346,
-                0.03703500747680664,
-                0.036560897827148435,
-                0.036534271240234374,
-                0.03651379013061523,
-                0.03656294250488281,
-                0.03655475234985352,
-                0.036588542938232424,
-                0.0366561279296875,
-                0.036604927062988284,
-                0.03645849609375,
-                0.03647385787963867,
-                0.036848960876464845,
-                0.03648819351196289,
-                0.03646361541748047,
-                0.03649945449829101,
-                0.03648409652709961,
-                0.036549633026123046,
-                0.03645747375488281,
-                0.03648409652709961,
-                0.03648716735839844,
-                0.03657318496704102,
-                0.036534271240234374,
-                0.036519935607910156,
-                0.036534271240234374,
-                0.03645542526245117,
-                0.03642675018310547,
-                0.03635734558105469,
-                0.03637452697753906,
-                0.03650979232788086,
-                0.036378623962402344,
-                0.03641241455078125,
-                0.03640217590332031,
-                0.03638579177856445,
-                0.036359169006347655,
-                0.03651891326904297,
-                0.03640524673461914,
-                0.036434112548828126,
-                0.03645951843261719,
-                0.03646057510375977,
-                0.03650457763671875,
-                0.03661414337158203,
-                0.03651891326904297,
-                0.036953086853027346,
-                0.03647385787963867,
-                0.036495361328125,
-                0.03657318496704102,
-                0.03652096176147461,
-                0.03665407943725586,
-                0.036536319732666016,
-                0.036490238189697266,
-                0.03649446487426758,
-                0.03645439910888672,
-                0.03660902404785156,
-                0.036587520599365236,
-                0.036482048034667966,
-                0.03690393447875977,
-                0.03634483337402344,
-                0.03654143905639649,
-                0.036604927062988284,
-                0.03653327941894531,
-                0.036480255126953125,
-                0.0365926399230957,
-                0.03654348754882813,
-                0.03650387191772461,
-                0.036416511535644534,
-                0.03654246520996094,
-                0.03647999954223633,
-                0.03659775924682617,
-                0.03640524673461914,
-                0.03647385787963867,
-                0.03642787170410156,
-                0.03655475234985352,
-                0.03646976089477539,
-                0.03649228668212891,
-                0.03646361541748047,
-                0.036503551483154296,
-                0.03650969696044922,
-                0.03644825744628906,
-                0.03646464157104492,
-                0.036466686248779294,
-                0.036549633026123046,
-                0.03653222274780273,
-                0.036536319732666016,
-                0.03662233734130859,
-                0.03646156692504883,
-                0.0365291519165039,
-                0.036503551483154296,
-                0.036452350616455076,
-                0.03648102569580078,
-                0.03645849609375,
-                0.03634688186645508,
-                0.0364400634765625,
-                0.03647488021850586,
-                0.03652403259277344,
-                0.03657625579833984,
-                0.03668787384033203,
-                0.03667660903930664,
-                0.03722137451171875,
-                0.03666124725341797,
-                0.036640800476074216,
-                0.03656294250488281,
-                0.03653222274780273,
-                0.03660287857055664,
-                0.036519935607910156,
-                0.03675852966308594,
-                0.03663872146606445,
-                0.03665100860595703,
-                0.03662131118774414,
-                0.036567039489746093,
-                0.036585472106933595,
-                0.03693363189697266,
-                0.036643905639648436,
-                0.03649638366699219,
-                0.03661721420288086,
-                0.0365333137512207,
-                0.03652096176147461,
-                0.03665510559082031,
-                0.036569087982177735,
-                0.036567039489746093,
-                0.036536319732666016,
-                0.03656192016601562,
-                0.03659366226196289,
-                0.03652505493164063,
-                0.036620288848876956,
-                0.03659366226196289,
-                0.03662847900390625,
-                0.03659980773925781,
-                0.03665510559082031,
-                0.036590591430664066,
-                0.03650048065185547,
-                0.036596736907958984,
-                0.03666022491455078,
-                0.036567039489746093,
-                0.03649228668212891,
-                0.0364769287109375,
-                0.03647078323364258,
-                0.036606975555419925,
-                0.036560897827148435,
-                0.03660083389282227,
-                0.0366192626953125,
-                0.03662233734130859,
-                0.03661004638671875,
-                0.03660595321655274,
-                0.036539390563964845,
-                0.03653017425537109,
-                0.03647084808349609,
-                0.036550655364990234,
-                0.03652505493164063,
-                0.03652710342407227,
-                0.036572158813476564,
-                0.0365588493347168,
-                0.03655987167358398,
-                0.03704934310913086,
-                0.036435966491699216,
-                0.03655475234985352,
-                0.03649638366699219,
-                0.03643494415283203,
-                0.03659366226196289,
-                0.03644825744628906,
-                0.036544513702392575,
-                0.03651401519775391,
-                0.03649638366699219,
-                0.03652403259277344,
-                0.036557823181152346,
-                0.03649433517456055,
-                0.03681382369995117,
-                0.03683647918701172,
-                0.03665407943725586,
-                0.036647937774658204,
-                0.03665203094482422,
-                0.03654655838012695,
-                0.03661721420288086,
-                0.0365926399230957,
-                0.036601856231689454,
-                0.03652403259277344,
-                0.03654364776611328,
-                0.03660287857055664,
-                0.03661414337158203,
-                0.036544513702392575,
-                0.03644313430786133,
-                0.036557823181152346,
-                0.036569087982177735,
-                0.03657625579833984,
-                0.03654860687255859,
-                0.03660083389282227,
-                0.03652608108520508,
-                0.03655987167358398,
-                0.036597953796386716,
-                0.03650457763671875,
-                0.036627521514892576,
-                0.036557823181152346,
-                0.036550655364990234,
-                0.03662847900390625,
-                0.03647999954223633,
-                0.03648531341552735,
-                0.03661414337158203,
-                0.036563968658447264,
-                0.03650969696044922,
-                0.03657625579833984,
-                0.03653222274780273,
-                0.036596736907958984,
-                0.036515071868896486,
-                0.036710624694824216,
-                0.03662556838989258,
-                0.03660800170898437,
-                0.03661004638671875,
-                0.03662745666503906,
-                0.036831230163574216,
-                0.03676364898681641,
-                0.036581569671630856,
-                0.03659775924682617,
-                0.03651071929931641,
-                0.03659775924682617,
-                0.03662643051147461,
-                0.036843521118164066,
-                0.03667763137817383,
-                0.03669708633422852,
-                0.03661004638671875,
-                0.036708351135253905,
-                0.03661414337158203,
-                0.036601856231689454,
-                0.03690086364746094,
-                0.03662233734130859,
-                0.036634624481201174,
-                0.03659775924682617,
-                0.03655168151855469,
-                0.036657150268554685,
-                0.03662438583374023,
-                0.036634624481201174,
-                0.036691967010498046,
-                0.0366110725402832,
-                0.03665100860595703,
-                0.036675582885742186,
-                0.03665206527709961,
-                0.03661209487915039,
-                0.03655168151855469,
-                0.036673534393310545,
-                0.036574207305908206,
-                0.03658444976806641,
-                0.03660800170898437,
-                0.03657625579833984,
-                0.036555774688720705,
-                0.03655987167358398,
-                0.036686847686767575,
-                0.036659198760986327,
-                0.03652403259277344,
-                0.03649331283569336,
-                0.03652403259277344
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 6984.6638067040985
         },
         "energy": null,
         "efficiency": null
@@ -297,23 +61,22 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 965.369856,
-            "max_vram": 2858.418176,
-            "max_reserved": 2373.976064,
-            "max_allocated": 2326.085632
         },
         "latency": {
             "unit": "s",
-            "mean": 7.126217727661133,
-            "stdev": 0.004621311187745114,
             "values": [
-                7.130839038848878,
-                7.121596416473388
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.78335798107764
         },
         "energy": null,
         "efficiency": null
@@ -322,524 +85,269 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.027945951873180924,
-            "stdev": 0.00010514274813064182,
             "values": [
-                0.02851430320739746,
-                0.028053504943847656,
-                0.027992063522338868,
-                0.028012544631958007,
-                0.027979776382446288,
-                0.028059648513793944,
-                0.028047359466552735,
-                0.028090368270874022,
-                0.02795417594909668,
-                0.02795929527282715,
-                0.027986944198608397,
-                0.02814361572265625,
-                0.028079103469848633,
-                0.028047359466552735,
-                0.02797772789001465,
-                0.027996160507202147,
-                0.02797670364379883,
-                0.027966463088989257,
-                0.028269567489624024,
-                0.028069887161254883,
-                0.02794086456298828,
-                0.028012544631958007,
-                0.028044288635253906,
-                0.028044288635253906,
-                0.027947008132934572,
-                0.02794598388671875,
-                0.02792755126953125,
-                0.028027904510498046,
-                0.02793984031677246,
-                0.028033023834228517,
-                0.02799001693725586,
-                0.028016639709472657,
-                0.02795315170288086,
-                0.028004352569580077,
-                0.02793574333190918,
-                0.02838528060913086,
-                0.028011520385742186,
-                0.028013568878173828,
-                0.02796441650390625,
-                0.028027904510498046,
-                0.0279552001953125,
-                0.0279552001953125,
-                0.027930624008178712,
-                0.028015615463256836,
-                0.027884544372558592,
-                0.028080127716064454,
-                0.027982847213745117,
-                0.028076032638549804,
-                0.028036096572875976,
-                0.028051456451416015,
-                0.02797875213623047,
-                0.028314624786376953,
-                0.02795622444152832,
-                0.028107776641845703,
-                0.028075008392333983,
-                0.028076032638549804,
-                0.02797670364379883,
-                0.028017663955688478,
-                0.027975679397583008,
-                0.028000255584716797,
-                0.027999391555786134,
-                0.02800111961364746,
-                0.02794803237915039,
-                0.027966463088989257,
-                0.02795622444152832,
-                0.02792140769958496,
-                0.02791219139099121,
-                0.02794086456298828,
-                0.02793574333190918,
-                0.027890687942504884,
-                0.027922431945800782,
-                0.02793164825439453,
-                0.027872255325317383,
-                0.027930624008178712,
-                0.027851776123046876,
-                0.02778828811645508,
-                0.02793267250061035,
-                0.027975679397583008,
-                0.028018688201904295,
-                0.02791935920715332,
-                0.027832319259643554,
-                0.027873279571533204,
-                0.027805696487426756,
-                0.02796134376525879,
-                0.027938816070556642,
-                0.027913215637207032,
-                0.02796544075012207,
-                0.027997184753417968,
-                0.02793779182434082,
-                0.028033023834228517,
-                0.028458208084106446,
-                0.02796316719055176,
-                0.028014591217041016,
-                0.028062719345092774,
-                0.027982847213745117,
-                0.027979776382446288,
-                0.02790399932861328,
-                0.02795110321044922,
-                0.02792140769958496,
-                0.02793471908569336,
-                0.02795008087158203,
-                0.02792038345336914,
-                0.02793984031677246,
-                0.027854848861694335,
-                0.027885568618774413,
-                0.02795008087158203,
-                0.02793369674682617,
-                0.0283637752532959,
-                0.027882495880126954,
-                0.02790399932861328,
-                0.027881471633911133,
-                0.027901952743530273,
-                0.027876352310180662,
-                0.028064767837524415,
-                0.02795315170288086,
-                0.02792550468444824,
-                0.027824352264404297,
-                0.028046112060546875,
-                0.027963392257690428,
-                0.027907072067260744,
-                0.02796544075012207,
-                0.02795417594909668,
-                0.027996160507202147,
-                0.0279685115814209,
-                0.027974655151367187,
-                0.028027904510498046,
-                0.0279685115814209,
-                0.02797875213623047,
-                0.0279552001953125,
-                0.027885568618774413,
-                0.028003328323364256,
-                0.027897855758666993,
-                0.027876352310180662,
-                0.02793369674682617,
-                0.02794598388671875,
-                0.028025856018066408,
-                0.027982847213745117,
-                0.028056575775146485,
-                0.027969568252563477,
-                0.02802070426940918,
-                0.028084224700927734,
-                0.02796953582763672,
-                0.028059648513793944,
-                0.027926528930664062,
-                0.027878400802612304,
-                0.02790297508239746,
-                0.027851808547973634,
-                0.027989984512329102,
-                0.027947008132934572,
-                0.028014591217041016,
-                0.0279685115814209,
-                0.028055551528930665,
-                0.027982847213745117,
-                0.028036096572875976,
-                0.028010496139526365,
-                0.028084224700927734,
-                0.027926528930664062,
-                0.027936767578125,
-                0.027871231079101562,
-                0.02791116714477539,
-                0.027837440490722655,
-                0.028017663955688478,
-                0.02831974411010742,
-                0.02788761520385742,
-                0.0279685115814209,
-                0.028015615463256836,
-                0.02794905662536621,
-                0.02797260856628418,
-                0.027852800369262694,
-                0.027991039276123047,
-                0.027827199935913087,
-                0.02794393539428711,
-                0.02794905662536621,
-                0.02795212745666504,
-                0.027915264129638673,
-                0.027820032119750978,
-                0.02789580726623535,
-                0.02790809631347656,
-                0.028039264678955077,
-                0.0283022403717041,
-                0.028025856018066408,
-                0.027963392257690428,
-                0.02798591995239258,
-                0.02793267250061035,
-                0.028004352569580077,
-                0.028041215896606447,
-                0.027926528930664062,
-                0.028054527282714844,
-                0.027926528930664062,
-                0.028017663955688478,
-                0.02800230407714844,
-                0.027986944198608397,
-                0.02795622444152832,
-                0.028059648513793944,
-                0.02791628837585449,
-                0.028058624267578124,
-                0.02793574333190918,
-                0.027983871459960938,
-                0.027852800369262694,
-                0.028069887161254883,
-                0.027824127197265625,
-                0.02792857551574707,
-                0.0279552001953125,
-                0.02795417594909668,
-                0.027874303817749024,
-                0.027967487335205078,
-                0.02788761520385742,
-                0.02798591995239258,
-                0.027872255325317383,
-                0.027891712188720705,
-                0.027844736099243164,
-                0.028102527618408202,
-                0.027870208740234374,
-                0.027894784927368164,
-                0.027808767318725586,
-                0.0279418888092041,
-                0.027842559814453126,
-                0.027847679138183593,
-                0.027874303817749024,
-                0.028008447647094727,
-                0.027875328063964845,
-                0.027864063262939453,
-                0.027820032119750978,
-                0.027901952743530273,
-                0.02778009605407715,
-                0.027836416244506838,
-                0.027846847534179688,
-                0.02790790367126465,
-                0.02788256072998047,
-                0.02787936019897461,
-                0.027865087509155274,
-                0.027888639450073242,
-                0.027835391998291017,
-                0.027873279571533204,
-                0.02822979164123535,
-                0.027913055419921874,
-                0.027812864303588865,
-                0.027907072067260744,
-                0.027844608306884764,
-                0.027847679138183593,
-                0.02778828811645508,
-                0.02791219139099121,
-                0.027826208114624024,
-                0.027875295639038088,
-                0.027814912796020507,
-                0.02790297508239746,
-                0.027794431686401368,
-                0.027862016677856444,
-                0.027849727630615235,
-                0.027848703384399414,
-                0.02792448043823242,
-                0.02840575981140137,
-                0.02791731262207031,
-                0.027835391998291017,
-                0.027866111755371094,
-                0.02796953582763672,
-                0.028040191650390626,
-                0.027855871200561523,
-                0.02792857551574707,
-                0.027940095901489256,
-                0.02794163131713867,
-                0.027936767578125,
-                0.028019712448120116,
-                0.02794393539428711,
-                0.028033023834228517,
-                0.027949087142944334,
-                0.0280534725189209,
-                0.027979776382446288,
-                0.028065792083740236,
-                0.027914239883422853,
-                0.028000255584716797,
-                0.028015615463256836,
-                0.028025856018066408,
-                0.02790297508239746,
-                0.028081151962280275,
-                0.027896831512451172,
-                0.027967487335205078,
-                0.027918336868286132,
-                0.028018688201904295,
-                0.027894784927368164,
-                0.028038143157958984,
-                0.028030975341796875,
-                0.028057600021362306,
-                0.02794291114807129,
-                0.028031999588012696,
-                0.02794803237915039,
-                0.028034048080444338,
-                0.02794291114807129,
-                0.028042240142822264,
-                0.027923456192016603,
-                0.02797056007385254,
-                0.02796031951904297,
-                0.027910144805908203,
-                0.027890687942504884,
-                0.027983871459960938,
-                0.027914239883422853,
-                0.028057600021362306,
-                0.02794598388671875,
-                0.02795929527282715,
-                0.027947008132934572,
-                0.028019712448120116,
-                0.02793779182434082,
-                0.027991039276123047,
-                0.027872255325317383,
-                0.02796544075012207,
-                0.028421119689941408,
-                0.027915264129638673,
-                0.02789990425109863,
-                0.02796441650390625,
-                0.027910207748413084,
-                0.027958208084106446,
-                0.027947008132934572,
-                0.028035072326660155,
-                0.027955392837524413,
-                0.028006208419799804,
-                0.02796236801147461,
-                0.02793164825439453,
-                0.027901952743530273,
-                0.028043264389038085,
-                0.028030975341796875,
-                0.02815795135498047,
-                0.02794304084777832,
-                0.028351360321044922,
-                0.02789990425109863,
-                0.0279736328125,
-                0.02795315170288086,
-                0.02795008087158203,
-                0.027966463088989257,
-                0.02793164825439453,
-                0.027876352310180662,
-                0.027893760681152343,
-                0.027889663696289063,
-                0.02790399932861328,
-                0.027841535568237305,
-                0.028011520385742186,
-                0.02779648017883301,
-                0.02789580726623535,
-                0.027847679138183593,
-                0.02794803237915039,
-                0.027824127197265625,
-                0.027881471633911133,
-                0.027782207489013673,
-                0.02783839988708496,
-                0.027926528930664062,
-                0.027849727630615235,
-                0.02777497673034668,
-                0.028005376815795898,
-                0.027830272674560546,
-                0.027913215637207032,
-                0.027854848861694335,
-                0.027816959381103516,
-                0.027790336608886718,
-                0.027884544372558592,
-                0.02774732780456543,
-                0.027794431686401368,
-                0.027819007873535157,
-                0.027858943939208985,
-                0.027828224182128908,
-                0.027842559814453126,
-                0.02778828811645508,
-                0.027854848861694335,
-                0.027828224182128908,
-                0.027910144805908203,
-                0.027678720474243163,
-                0.027768831253051757,
-                0.027813888549804686,
-                0.027811840057373048,
-                0.027732992172241212,
-                0.027830272674560546,
-                0.027778047561645508,
-                0.027891712188720705,
-                0.02775961685180664,
-                0.02787942314147949,
-                0.027828224182128908,
-                0.027886592864990234,
-                0.027844608306884764,
-                0.027874303817749024,
-                0.028201984405517577,
-                0.027859968185424806,
-                0.027870208740234374,
-                0.02792140769958496,
-                0.027871231079101562,
-                0.027872255325317383,
-                0.027874303817749024,
-                0.027863040924072265,
-                0.02792140769958496,
-                0.02793984031677246,
-                0.02791731262207031,
-                0.027900928497314452,
-                0.027909120559692382,
-                0.027991039276123047,
-                0.02793369674682617,
-                0.027936767578125,
-                0.027870208740234374,
-                0.02834739112854004,
-                0.027858943939208985,
-                0.02791731262207031,
-                0.02793984031677246,
-                0.027853824615478515,
-                0.027831296920776367,
-                0.02792140769958496,
-                0.027852800369262694,
-                0.02794495964050293,
-                0.027867136001586915,
-                0.027859968185424806,
-                0.027881471633911133,
-                0.02774835205078125,
-                0.02775040054321289,
-                0.027817983627319336,
-                0.027842559814453126,
-                0.027857919692993165,
-                0.027826175689697266,
-                0.027846656799316406,
-                0.027896831512451172,
-                0.027837440490722655,
-                0.027893760681152343,
-                0.02796544075012207,
-                0.027855871200561523,
-                0.027901119232177734,
-                0.027956031799316407,
-                0.02798080062866211,
-                0.027853824615478515,
-                0.027930624008178712,
-                0.028027904510498046,
-                0.02793779182434082,
-                0.028036096572875976,
-                0.02799411201477051,
-                0.028089344024658205,
-                0.02798899269104004,
-                0.028070911407470703,
-                0.027936767578125,
-                0.027858943939208985,
-                0.027915264129638673,
-                0.02794825553894043,
-                0.027857696533203125,
-                0.02788256072998047,
-                0.027915199279785155,
-                0.02790399932861328,
-                0.027913215637207032,
-                0.027971584320068358,
-                0.027892736434936522,
-                0.02796352005004883,
-                0.028015487670898436,
-                0.027878400802612304,
-                0.02798182487487793,
-                0.02789580726623535,
-                0.02797772789001465,
-                0.027877376556396483,
-                0.0279736328125,
-                0.028184576034545897,
-                0.027876352310180662,
-                0.02790399932861328,
-                0.0279552001953125,
-                0.027821056365966795,
-                0.027910144805908203,
-                0.02772275161743164,
-                0.02792448043823242,
-                0.0277576961517334,
-                0.027983743667602538,
-                0.02776268768310547,
-                0.02795212745666504,
-                0.027847679138183593,
-                0.02793471908569336,
-                0.02795315170288086,
-                0.027999231338500977,
-                0.02793369674682617,
-                0.028253183364868165,
-                0.028067840576171874,
-                0.027971584320068358,
-                0.02792140769958496,
-                0.02793779182434082,
-                0.02794905662536621,
-                0.028054527282714844,
-                0.027885663986206056,
-                0.028016544342041014,
-                0.02796143913269043,
-                0.028012447357177735,
-                0.027812864303588865,
-                0.028022783279418945,
-                0.027809791564941407,
-                0.02792755126953125,
-                0.027791391372680663,
-                0.02799510383605957,
-                0.027941984176635744,
-                0.028007328033447267,
-                0.027883520126342775,
-                0.028047359466552735,
-                0.027828224182128908,
-                0.028020736694335937,
-                0.027785215377807617,
-                0.028009471893310548,
-                0.02792348861694336,
-                0.02804528045654297,
-                0.027849727630615235,
-                0.02799001693725586,
-                0.02791116714477539,
-                0.028140544891357422,
-                0.028029951095581054,
-                0.027997184753417968,
-                0.027821056365966795,
-                0.028015615463256836,
-                0.027857919692993165,
-                0.02793369674682617,
-                0.027864063262939453,
-                0.02794291114807129,
-                0.02791219139099121,
-                0.028023807525634766,
-                0.027868160247802733,
-                0.02793164825439453,
-                0.027850751876831056
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.78335798107763
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1990.262784,
+            "max_vram": 5314.183168,
+            "max_reserved": 4829.741056,
+            "max_allocated": 4586.114048
         },
         "latency": {
             "unit": "s",
+            "mean": 0.2717731669657939,
+            "stdev": 0.001455918787921021,
             "values": [
+                0.2787281494140625,
+                0.2732472229003906,
+                0.2723811950683594,
+                0.27061044311523436,
+                0.2724352111816406,
+                0.27068209838867185,
+                0.2723665771484375,
+                0.2706483154296875,
+                0.27239117431640625,
+                0.270740478515625,
+                0.27237069702148436,
+                0.2707220458984375,
+                0.27236865234375,
+                0.2706503601074219,
+                0.27236865234375,
+                0.2706872253417969,
+                0.2724403076171875,
+                0.27066265869140627,
+                0.27234100341796874,
+                0.2706042785644531,
+                0.27239321899414065,
+                0.2705940551757812,
+                0.2723061828613281,
+                0.2706431884765625,
+                0.272385009765625,
+                0.27065240478515623,
+                0.272321533203125,
+                0.2705950622558594,
+                0.27234817504882813,
+                0.2705920104980469,
+                0.27228671264648435,
+                0.27079473876953125,
+                0.2723460998535156,
+                0.2705797119140625,
+                0.2723819580078125,
+                0.2705809326171875,
+                0.2723594360351563
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 941.9620150808371
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 2000.748544,
+            "max_vram": 5708.447744,
+            "max_reserved": 5224.005632,
+            "max_allocated": 5142.572544
         },
         "latency": {
             "unit": "s",
+            "mean": 67.67080883789065,
+            "stdev": 0,
             "values": [
+                67.67080883789065
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 3.768242235893283
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.2653757209329045,
+            "stdev": 0.0008322589023743215,
             "values": [
+                0.26587136840820313,
+                0.2652405700683594,
+                0.2654105529785156,
+                0.26491494750976563,
+                0.26523443603515623,
+                0.26365850830078125,
+                0.26560818481445314,
+                0.26561944580078123,
+                0.26569830322265625,
+                0.26570855712890623,
+                0.26507672119140624,
+                0.26551910400390627,
+                0.2650214538574219,
+                0.26569317626953126,
+                0.26495281982421875,
+                0.26590924072265626,
+                0.265017333984375,
+                0.26585806274414064,
+                0.2650798034667969,
+                0.2656368713378906,
+                0.2648832092285156,
+                0.26548019409179685,
+                0.26523751831054687,
+                0.265491455078125,
+                0.2649190368652344,
+                0.26366156005859376,
+                0.26479617309570314,
+                0.2639718322753906,
+                0.26558770751953126,
+                0.2638919677734375,
+                0.26530712890625,
+                0.2646446228027344,
+                0.26524465942382813,
+                0.26387762451171876,
+                0.26531329345703125,
+                0.26394622802734374,
+                0.26527847290039064,
+                0.26381005859375,
+                0.26559384155273436,
+                0.26385614013671876,
+                0.2652600402832031,
+                0.26473779296875,
+                0.26482891845703127,
+                0.26301336669921876,
+                0.26571060180664063,
+                0.262982666015625,
+                0.26509823608398436,
+                0.265860107421875,
+                0.265248779296875,
+                0.26608026123046874,
+                0.26562969970703126,
+                0.26561126708984373,
+                0.2656450500488281,
+                0.2654484558105469,
+                0.2654310302734375,
+                0.2656942138671875,
+                0.2656942138671875,
+                0.2655836181640625,
+                0.2654310302734375,
+                0.26579864501953127,
+                0.2653378601074219,
+                0.26532965087890625,
+                0.2654515075683594,
+                0.265385986328125,
+                0.2655887451171875,
+                0.26581607055664064,
+                0.2650521545410156,
+                0.2653040771484375,
+                0.266144775390625,
+                0.2654228515625,
+                0.2661693420410156,
+                0.26313421630859374,
+                0.2661539916992188,
+                0.2635643005371094,
+                0.26595123291015627,
+                0.2650746765136719,
+                0.2655897521972656,
+                0.2640824279785156,
+                0.2657525634765625,
+                0.2642800598144531,
+                0.26538189697265624,
+                0.26611712646484376,
+                0.2657679443359375,
+                0.26585906982421875,
+                0.26564404296875,
+                0.2657914733886719,
+                0.26576895141601564,
+                0.26541671752929685,
+                0.2656737365722656,
+                0.263372802734375,
+                0.26533377075195314,
+                0.2642421875,
+                0.26569012451171875,
+                0.26548736572265624,
+                0.265744384765625,
+                0.2659727478027344,
+                0.2708408203125,
+                0.26579864501953127,
+                0.26518936157226564,
+                0.2661273498535156,
+                0.26528460693359374,
+                0.26622872924804686,
+                0.2656542663574219,
+                0.26621746826171877,
+                0.2655733642578125,
+                0.2637916259765625,
+                0.26576589965820313,
+                0.2658396301269531,
+                0.26570648193359375,
+                0.2659154052734375,
+                0.26574746704101565,
+                0.2642483215332031,
+                0.26578125,
+                0.26380697631835937,
+                0.26570965576171873,
+                0.26401681518554687,
+                0.2659287109375,
+                0.26403021240234376,
+                0.26580990600585935,
+                0.26391961669921876,
+                0.2655201416015625,
+                0.2642012023925781,
+                0.265591796875,
+                0.26412033081054687,
+                0.2655211486816406,
+                0.26415924072265623,
+                0.26419711303710935,
+                0.264205322265625,
+                0.26451763916015625,
+                0.2646148986816406,
+                0.2641827697753906,
+                0.2647930908203125,
+                0.2643814392089844,
+                0.26490469360351565,
+                0.2642401428222656,
+                0.2648279113769531,
+                0.2642933654785156,
+                0.26478387451171875,
+                0.264121337890625,
+                0.264153076171875,
+                0.26442547607421873,
+                0.26408551025390625,
+                0.2641848449707031,
+                0.26420327758789064,
+                0.26429440307617186,
+                0.2644817810058594,
+                0.26437939453125,
+                0.26429644775390626,
+                0.2657976379394531,
+                0.2639544372558594,
+                0.26541259765625,
+                0.2666690673828125,
+                0.26544024658203125,
+                0.26617138671875,
+                0.2641448974609375,
+                0.2642217102050781,
+                0.2657873840332031,
+                0.26414080810546875,
+                0.26605978393554686,
+                0.26521804809570315,
+                0.2661335144042969,
+                0.2658672790527344,
+                0.2659952697753906,
+                0.2662533264160156,
+                0.2658058166503906,
+                0.2677821350097656,
+                0.26605465698242187,
+                0.2660433959960937,
+                0.2653552551269531,
+                0.2660505676269531,
+                0.26535833740234377,
+                0.2661560363769531,
+                0.266787841796875,
+                0.2642288513183594,
+                0.26611508178710935,
+                0.2644664306640625,
+                0.26595431518554685,
+                0.26437734985351563,
+                0.2658559875488281,
+                0.2644961242675781,
+                0.26560614013671874,
+                0.2656624755859375,
+                0.2661109619140625,
+                0.2660997009277344,
+                0.26584063720703127,
+                0.26598501586914064,
+                0.2658682861328125,
+                0.2657177734375,
+                0.2657198181152344,
+                0.26573312377929686,
+                0.26561944580078123,
+                0.26570443725585935,
+                0.2656942138671875,
+                0.26558566284179685,
+                0.26561126708984373,
+                0.2656204833984375,
+                0.2659757995605469,
+                0.26554266357421874,
+                0.2657925109863281,
+                0.2659860534667969,
+                0.26586932373046873,
+                0.2660341796875,
+                0.265818115234375,
+                0.26580172729492185,
+                0.26587954711914064,
+                0.2655907897949219,
+                0.26608538818359373,
+                0.2656163940429688,
+                0.2660034484863281,
+                0.2660106201171875,
+                0.26602392578125,
+                0.2659921875,
+                0.2659287109375,
+                0.26578021240234373,
+                0.26561126708984373,
+                0.2659471435546875,
+                0.265449462890625,
+                0.2651197509765625,
+                0.26594305419921876,
+                0.2654003295898438,
+                0.2656102294921875,
+                0.26606591796875,
+                0.2654761047363281,
+                0.2660157470703125,
+                0.2655703125,
+                0.26607513427734375,
+                0.2656870422363281,
+                0.26564199829101565,
+                0.26581402587890623,
+                0.2658805847167969,
+                0.2658365478515625,
+                0.26573516845703127,
+                0.26693426513671875,
+                0.26610995483398436,
+                0.2658529357910156,
+                0.2660577392578125,
+                0.2655909423828125,
+                0.2661978454589844,
+                0.2657853393554688,
+                0.2661549987792969,
+                0.26549554443359374,
+                0.2652876892089844,
+                0.26575872802734374,
+                0.26558670043945315,
+                0.26607000732421876,
+                0.2655959167480469,
+                0.2655467529296875,
+                0.2661314697265625,
+                0.26560614013671874,
+                0.2661109619140625,
+                0.26558978271484374,
+                0.26623074340820313,
+                0.26575872802734374,
+                0.26574542236328125,
+                0.2658887634277344
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 3.768242235893283
         },
         "energy": null,
         "efficiency": null