kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Mar 14, 2024

Commit

9b3b4b3

verified ·

1 Parent(s): 8fc2a2d

Upload /Llama-2-7b-hf/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Llama-2-7b-hf/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +722 -722

Llama-2-7b-hf/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,258 +2,258 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 982.437888,
             "max_vram": 4905.238528,
             "max_reserved": 4420.796416,
             "max_allocated": 4252.667392
         },
         "latency": {
             "unit": "s",
-            "mean": 0.042258727737619926,
-            "stdev": 0.000582065656107356,
             "values": [
-                0.047258270263671874,
-                0.04455014419555664,
-                0.04322304153442383,
-                0.043133953094482425,
-                0.04293443298339844,
-                0.04335615921020508,
-                0.04328345489501953,
-                0.04333465576171875,
-                0.04698521423339844,
-                0.04396953582763672,
-                0.04190105438232422,
-                0.04237209701538086,
-                0.0419502067565918,
-                0.042387454986572266,
-                0.041899009704589846,
-                0.042355712890625,
-                0.041852928161621096,
                 0.042379264831542966,
-                0.04192870330810547,
-                0.04236492919921875,
-                0.04185190582275391,
                 0.04237209701538086,
-                0.04184371185302734,
-                0.04237823867797851,
-                0.04193689727783203,
-                0.04254924774169922,
-                0.04188979339599609,
-                0.04238643264770508,
-                0.04189491271972656,
                 0.042365951538085936,
-                0.04185513687133789,
                 0.04237516784667969,
-                0.04209664154052734,
                 0.04235161590576172,
-                0.04186111831665039,
-                0.04240486526489258,
-                0.04228607940673828,
-                0.04238336181640625,
-                0.0418785285949707,
-                0.0424007682800293,
-                0.04184678268432617,
-                0.042349567413330076,
-                0.041859073638916014,
-                0.04237020874023437,
-                0.04184473419189453,
                 0.042382335662841795,
-                0.04188159942626953,
-                0.04237120056152344,
-                0.041855998992919925,
-                0.04238336181640625,
-                0.041867263793945314,
-                0.042355712890625,
-                0.04187648010253906,
-                0.042349567413330076,
-                0.041847808837890625,
-                0.042398719787597655,
-                0.04206796646118164,
-                0.04261478424072265,
-                0.04187955093383789,
-                0.04237721633911133,
-                0.041864192962646485,
-                0.042363903045654294,
-                0.04187750244140625,
-                0.042398719787597655,
-                0.04185804748535156,
-                0.04237635040283203,
-                0.04187238311767578,
-                0.04236207962036133,
-                0.041864192962646485,
-                0.042355712890625,
-                0.04189184188842773,
-                0.042387454986572266,
-                0.04232294464111328,
-                0.04237311935424805,
-                0.041866241455078126,
-                0.042369022369384765,
-                0.04188671875,
-                0.042343425750732425,
-                0.04193689727783203,
-                0.0423454704284668,
-                0.04187955093383789,
-                0.042349567413330076,
-                0.04190003204345703,
-                0.042355712890625,
-                0.04231270217895508,
-                0.04235673522949219,
-                0.04196681594848633,
-                0.04238438415527344,
-                0.041866241455078126,
-                0.042369022369384765,
-                0.04189388656616211,
-                0.0423785285949707,
-                0.04193689727783203,
-                0.042359809875488284,
-                0.0419317741394043,
-                0.042379264831542966,
-                0.04190719985961914,
-                0.04236185455322266,
-                0.04187340927124023,
-                0.04235161590576172,
-                0.04186111831665039,
-                0.04235366439819336,
-                0.04187443161010742,
-                0.042347518920898435,
-                0.04185190582275391,
-                0.04238556671142578,
-                0.04185702514648437,
-                0.04236492919921875,
-                0.04186316680908203,
-                0.042363903045654294,
-                0.041856224060058594,
-                0.04234649658203125,
-                0.0418950080871582,
-                0.04234239959716797,
-                0.041918464660644535,
-                0.042369022369384765,
-                0.04185804748535156,
                 0.04236083221435547,
-                0.04193280029296875,
-                0.04235776138305664,
-                0.04231475067138672,
-                0.0423741455078125,
-                0.0419420166015625,
                 0.04237619018554688,
-                0.04199321746826172,
-                0.04238131332397461,
                 0.04203212738037109,
                 0.042393600463867184,
-                0.04202598571777344,
                 0.04235776138305664,
-                0.04187750244140625,
                 0.04236083221435547,
-                0.04218265533447266,
                 0.042385406494140625,
-                0.041952255249023435,
-                0.04235776138305664,
-                0.041902080535888675,
-                0.04237516784667969,
-                0.0418600959777832,
-                0.04236697769165039,
-                0.042014720916748044,
-                0.04235161590576172,
-                0.04187033462524414,
-                0.04235673522949219,
-                0.04192256164550781,
                 0.04237721633911133,
-                0.04187648010253906,
-                0.042343425750732425,
-                0.0419051513671875,
-                0.0423741455078125,
-                0.04198604965209961,
-                0.04236288070678711,
-                0.04187648010253906,
-                0.04237516784667969,
-                0.04206694412231445,
                 0.04235161590576172,
-                0.04191436767578125,
-                0.044368896484375,
-                0.04288819122314453,
                 0.0423741455078125,
-                0.041855998992919925,
-                0.04235468673706055,
-                0.041896961212158204,
-                0.04238336181640625,
-                0.04206284713745117,
-                0.042355712890625,
-                0.04192051315307617,
-                0.04234035110473633,
-                0.04248473739624024,
-                0.04237619018554688,
-                0.04198809432983398,
-                0.042365951538085936,
-                0.042014942169189454,
-                0.04236083221435547,
-                0.04184473419189453,
-                0.04235366439819336,
-                0.04186316680908203,
-                0.042401790618896484,
-                0.042354782104492186,
                 0.04240691375732422,
-                0.04187136077880859,
-                0.042374305725097657,
-                0.04206911849975586,
-                0.042365119934082034,
-                0.04197580718994141,
                 0.042363903045654294,
-                0.04193382263183594,
-                0.042387454986572266,
-                0.04192563247680664,
-                0.04235878372192383,
-                0.04187750244140625,
-                0.04238336181640625,
-                0.04209356689453125,
-                0.04237619018554688,
-                0.041957374572753905,
-                0.04233932876586914,
-                0.04196454238891602,
                 0.04238848114013672,
-                0.04195635223388672,
-                0.04237311935424805,
-                0.04188979339599609,
-                0.04236185455322266,
-                0.041968639373779294,
-                0.04237619018554688,
-                0.041902080535888675,
-                0.04238051223754883,
-                0.042036479949951175,
-                0.042349567413330076,
-                0.041918464660644535,
-                0.04236185455322266,
-                0.041968639373779294,
-                0.04236083221435547,
-                0.041908222198486327,
-                0.04236185455322266,
-                0.041855998992919925,
                 0.042379264831542966,
-                0.042425342559814457,
                 0.04237516784667969,
-                0.04187750244140625,
-                0.042369281768798825,
-                0.04196044921875,
-                0.04237619018554688,
-                0.04188703918457031,
-                0.04237619018554688,
-                0.04194105529785156,
                 0.042382335662841795,
-                0.042172416687011716,
-                0.04245196914672852,
-                0.04191027069091797,
-                0.0423454704284668,
-                0.042281982421875,
-                0.04236697769165039,
-                0.04186316680908203,
-                0.042355712890625,
-                0.04190310287475586,
                 0.04237619018554688,
-                0.04189593505859375
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 6057.920190817801
         },
         "energy": null,
         "efficiency": null
@@ -261,23 +261,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 982.437888,
             "max_vram": 5255.462912,
             "max_reserved": 4771.0208,
-            "max_allocated": 4715.611648
         },
         "latency": {
             "unit": "s",
-            "mean": 8.489722272872921,
-            "stdev": 0.012863920211795055,
             "values": [
-                8.476858352661127,
-                8.502586193084717
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 30.03631824503819
         },
         "energy": null,
         "efficiency": null
@@ -286,524 +286,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.03329302852107029,
-            "stdev": 0.0001376279300769796,
             "values": [
-                0.03315507125854492,
-                0.03353497695922852,
-                0.03332505416870117,
-                0.03326591873168945,
-                0.033143550872802734,
-                0.033260639190673826,
-                0.03330140686035156,
-                0.03323904037475586,
-                0.03323494338989258,
-                0.033186817169189455,
-                0.033143806457519534,
-                0.03326668930053711,
-                0.03321139144897461,
-                0.03326976013183594,
-                0.03401318359375,
-                0.0333007698059082,
-                0.03305648040771485,
-                0.033040382385253905,
-                0.033113086700439456,
-                0.03319091033935547,
-                0.03304243087768555,
-                0.03308236694335937,
-                0.03324518585205078,
-                0.03311820983886719,
-                0.033124351501464845,
-                0.03313868713378906,
-                0.03303833770751953,
-                0.03317452621459961,
-                0.03328921508789062,
-                0.03322265625,
-                0.033165313720703124,
-                0.03322367858886719,
-                0.03372851181030274,
-                0.033185791015625,
-                0.03314995193481445,
-                0.03333529663085937,
-                0.033186817169189455,
-                0.03325030517578125,
-                0.03328921508789062,
-                0.033260543823242186,
-                0.0332410888671875,
-                0.03316534423828125,
-                0.03322364807128906,
-                0.03316633605957031,
-                0.033172481536865236,
-                0.03330998229980469,
-                0.03313123321533203,
-                0.03381862258911133,
-                0.03307110214233398,
-                0.0331960334777832,
-                0.03316659164428711,
-                0.033193824768066406,
-                0.03315292739868164,
-                0.03325747299194336,
-                0.03307622528076172,
-                0.033031169891357424,
-                0.03310899353027344,
-                0.03341843032836914,
-                0.03297977447509766,
-                0.03315302276611328,
-                0.033127422332763674,
-                0.03314483261108398,
-                0.03308342361450195,
-                0.033077217102050784,
-                0.033291263580322264,
-                0.03320115280151367,
-                0.033083393096923826,
-                0.033178752899169925,
-                0.03321433639526367,
-                0.033209342956542966,
-                0.03310182571411133,
-                0.03337625503540039,
-                0.03318476867675781,
-                0.033438751220703125,
-                0.03312432098388672,
-                0.03304243087768555,
-                0.03305267333984375,
-                0.033143806457519534,
-                0.033058815002441407,
-                0.03324825668334961,
-                0.03322777557373047,
-                0.033233951568603516,
-                0.03321136093139648,
-                0.03331071853637695,
-                0.033350654602050785,
-                0.033279998779296875,
-                0.03313151931762695,
-                0.03329740905761719,
-                0.03326464080810547,
-                0.03327385711669922,
-                0.03329536056518555,
-                0.03327391815185547,
-                0.03313555145263672,
-                0.03367424011230469,
-                0.0333199348449707,
-                0.03324518585205078,
-                0.033175552368164066,
-                0.03320124816894531,
-                0.03319696044921875,
-                0.03317657470703125,
-                0.03317145538330078,
-                0.03331071853637695,
-                0.033339393615722655,
-                0.03322060775756836,
-                0.03315507125854492,
-                0.03337539291381836,
-                0.0333812141418457,
-                0.03341209411621094,
-                0.033486846923828126,
-                0.03326464080810547,
-                0.03314995193481445,
-                0.03321343994140625,
-                0.03326668930053711,
-                0.033337345123291014,
-                0.0335093765258789,
-                0.03333657455444336,
-                0.03333504104614258,
-                0.03328409576416016,
-                0.03325132751464844,
-                0.0332861442565918,
-                0.03321446228027344,
-                0.033334270477294925,
-                0.03328409576416016,
-                0.033364158630371094,
-                0.03328799819946289,
-                0.033396736145019534,
-                0.03326566314697266,
-                0.03334041595458984,
-                0.033331199645996096,
-                0.03330355072021484,
-                0.033160190582275394,
-                0.03320537567138672,
-                0.03315596771240235,
-                0.03327283096313476,
-                0.03318067169189453,
-                0.033323009490966796,
-                0.03323187255859375,
-                0.03327385711669922,
-                0.03320729446411133,
-                0.03323289489746094,
-                0.033293312072753906,
-                0.03315302276611328,
-                0.03324620819091797,
-                0.033219585418701174,
-                0.033192958831787106,
-                0.03319091033935547,
-                0.0331776008605957,
-                0.033271808624267575,
-                0.033135616302490234,
-                0.033175552368164066,
-                0.033175552368164066,
-                0.033323009490966796,
-                0.033119232177734374,
-                0.03321753692626953,
-                0.03366400146484375,
-                0.033230846405029296,
-                0.03325132751464844,
-                0.033462272644042966,
-                0.033230846405029296,
-                0.03319091033935547,
-                0.03315609741210938,
-                0.03331379318237305,
-                0.0332042236328125,
-                0.033328128814697267,
-                0.033301502227783206,
-                0.03336908721923828,
-                0.03324313735961914,
-                0.03333529663085937,
-                0.033667072296142575,
-                0.03329740905761719,
                 0.033463294982910154,
-                0.033235233306884764,
-                0.033362655639648436,
-                0.03308544158935547,
-                0.03311718368530273,
-                0.03316428756713867,
-                0.03306496047973633,
-                0.03307827377319336,
-                0.0331141128540039,
-                0.03307417678833008,
-                0.03305984115600586,
-                0.03318067169189453,
-                0.03315507125854492,
-                0.032982017517089846,
-                0.03300864028930664,
-                0.03314790344238281,
-                0.03309875106811523,
-                0.033007614135742186,
-                0.03304857635498047,
-                0.03325747299194336,
-                0.03307417678833008,
-                0.033170433044433595,
-                0.03322675323486328,
-                0.03318476867675781,
-                0.03311206436157227,
-                0.033188865661621096,
-                0.03305984115600586,
-                0.03315609741210938,
-                0.033132545471191405,
-                0.0331776008605957,
                 0.033438720703125,
-                0.03308441543579101,
-                0.033091583251953126,
-                0.033107967376708985,
-                0.03325439834594727,
-                0.033209342956542966,
-                0.0331253776550293,
-                0.03307417678833008,
-                0.033068031311035154,
-                0.0332564468383789,
-                0.03313868713378906,
-                0.033142784118652346,
-                0.03304652786254883,
-                0.033113086700439456,
-                0.03360665512084961,
-                0.033410049438476565,
-                0.033119232177734374,
-                0.03349200057983399,
-                0.033191902160644535,
-                0.03338979339599609,
-                0.033137439727783206,
-                0.033320960998535154,
-                0.03320627212524414,
-                0.03322367858886719,
-                0.033279998779296875,
-                0.03355347061157227,
-                0.03323897552490234,
-                0.03323801422119141,
-                0.033235969543457033,
-                0.03385036849975586,
-                0.033307647705078124,
-                0.03336601638793945,
-                0.03354521560668945,
                 0.033358848571777344,
-                0.03334348678588867,
-                0.03326985549926758,
-                0.03338435363769531,
-                0.0332410888671875,
-                0.03324313735961914,
-                0.0332564468383789,
-                0.03326976013183594,
-                0.03331398391723633,
-                0.03333407974243164,
-                0.033331199645996096,
-                0.033379520416259766,
-                0.033417022705078125,
-                0.0332492790222168,
-                0.033432575225830076,
-                0.03322470474243164,
-                0.03325337600708008,
-                0.03328409576416016,
-                0.03337955093383789,
-                0.03337705612182617,
-                0.03328409576416016,
-                0.03338547134399414,
-                0.03331875228881836,
-                0.033247230529785156,
-                0.03322060775756836,
-                0.03324739074707031,
-                0.0332655029296875,
-                0.03320729446411133,
-                0.033255424499511715,
-                0.03323113632202149,
-                0.03315683364868164,
-                0.03327385711669922,
-                0.033242111206054685,
-                0.03320012664794922,
-                0.03324620819091797,
-                0.033268798828125,
-                0.03313657760620117,
-                0.0332760009765625,
-                0.03317238235473633,
-                0.03328118515014648,
-                0.033259361267089844,
-                0.03370195388793945,
-                0.033266624450683596,
-                0.03314176177978516,
-                0.03318374252319336,
-                0.033308830261230465,
-                0.03329315185546875,
-                0.03330047988891602,
-                0.0333199348449707,
-                0.03330876922607422,
-                0.033288097381591795,
-                0.03336294555664063,
-                0.03331174468994141,
-                0.03337830352783203,
-                0.03330252838134766,
-                0.033643585205078125,
-                0.033210304260253905,
-                0.03321343994140625,
-                0.03325439834594727,
-                0.03336908721923828,
-                0.033261566162109374,
-                0.033315135955810544,
-                0.03331961441040039,
-                0.03327283096313476,
-                0.03336191940307617,
-                0.03353497695922852,
-                0.03325132751464844,
-                0.033312767028808594,
-                0.033263614654541016,
-                0.03340800094604492,
-                0.03316428756713867,
-                0.03330047988891602,
-                0.03322982406616211,
-                0.03336115264892578,
-                0.033366783142089844,
-                0.03330047988891602,
-                0.03331686401367188,
-                0.033428577423095705,
-                0.03340995025634766,
                 0.033363967895507815,
-                0.0332042236328125,
-                0.03325337600708008,
-                0.03333017730712891,
-                0.033261566162109374,
-                0.03321036911010742,
-                0.03328102493286133,
-                0.03342950439453125,
-                0.033328128814697267,
-                0.033296382904052735,
-                0.03334873580932617,
-                0.033309566497802735,
                 0.033329151153564454,
-                0.033331199645996096,
                 0.03341209411621094,
-                0.03345510482788086,
-                0.033347583770751955,
-                0.03336703872680664,
-                0.03339571380615235,
-                0.03342335891723633,
-                0.0333199348449707,
-                0.0333199348449707,
-                0.03384012985229492,
-                0.03338444900512695,
-                0.03344179153442383,
-                0.03342233657836914,
-                0.03335372924804687,
-                0.03339980697631836,
-                0.03345510482788086,
-                0.03331379318237305,
-                0.03341721725463867,
-                0.03334860610961914,
-                0.03377260971069336,
-                0.033310657501220704,
-                0.03338751983642578,
-                0.033342464447021485,
-                0.03351551818847656,
-                0.033529857635498046,
-                0.03349708938598633,
-                0.033317886352539065,
-                0.03347260665893555,
-                0.033340320587158204,
-                0.03333529663085937,
-                0.03331379318237305,
-                0.03337420654296875,
-                0.03335782241821289,
-                0.033430782318115235,
-                0.03330227279663086,
-                0.03347251129150391,
-                0.033344512939453126,
                 0.03339571380615235,
-                0.033277950286865234,
-                0.033274879455566404,
-                0.033329151153564454,
-                0.03336191940307617,
-                0.03326259231567383,
-                0.03332710266113281,
-                0.03323904037475586,
-                0.03338854217529297,
-                0.033285118103027346,
-                0.03341721725463867,
-                0.03327078247070313,
-                0.03327078247070313,
                 0.033312767028808594,
-                0.033372161865234375,
-                0.0333834228515625,
                 0.0334284782409668,
-                0.033339393615722655,
                 0.03341107177734375,
-                0.03328841781616211,
-                0.03335248184204102,
-                0.033309696197509765,
-                0.033230846405029296,
-                0.033344512939453126,
-                0.03335372924804687,
-                0.03332505416870117,
-                0.03328204727172852,
-                0.03323699188232422,
-                0.033533119201660154,
-                0.03339961624145508,
-                0.03322982406616211,
-                0.03334860610961914,
-                0.03352371215820313,
-                0.033620990753173825,
-                0.03331071853637695,
-                0.033285118103027346,
-                0.03344076919555664,
                 0.033430526733398434,
-                0.03321651077270508,
-                0.03342335891723633,
-                0.03354521560668945,
-                0.03337932968139649,
-                0.03327590560913086,
-                0.033304576873779294,
-                0.03339980697631836,
-                0.033258495330810545,
-                0.0334919662475586,
-                0.03374387359619141,
-                0.033426433563232424,
-                0.03347148895263672,
-                0.03333126449584961,
-                0.03327686309814453,
-                0.033350654602050785,
-                0.033342464447021485,
-                0.03321241760253906,
-                0.033290241241455076,
-                0.03330572891235352,
-                0.033309566497802735,
-                0.03338956832885742,
-                0.03322777557373047,
-                0.033486846923828126,
-                0.03322777557373047,
-                0.03335391998291016,
-                0.033289024353027344,
-                0.033495166778564456,
-                0.033320831298828124,
-                0.0332861442565918,
-                0.033309696197509765,
-                0.03337625503540039,
-                0.03327385711669922,
-                0.0333199348449707,
-                0.033288192749023435,
-                0.03337113571166992,
-                0.033271808624267575,
-                0.03324620819091797,
-                0.03320012664794922,
-                0.033274879455566404,
-                0.03336806488037109,
-                0.03329228973388672,
-                0.03322470474243164,
-                0.03323904037475586,
-                0.03333529663085937,
-                0.03325747299194336,
-                0.033432575225830076,
-                0.033317886352539065,
-                0.03331379318237305,
-                0.03335168075561523,
-                0.033375232696533204,
-                0.033345535278320314,
-                0.03330559921264648,
-                0.03334656143188477,
-                0.03329536056518555,
-                0.033301502227783206,
-                0.03385036849975586,
-                0.03324415969848633,
-                0.03343462371826172,
-                0.03332710266113281,
-                0.03319091033935547,
-                0.03343360137939453,
-                0.03332403182983398,
-                0.03321036911010742,
-                0.03340185546875,
-                0.033320960998535154,
-                0.03324518585205078,
-                0.033432575225830076,
-                0.033326080322265625,
-                0.03335168075561523,
-                0.03380223846435547,
-                0.03327590560913086,
-                0.033306625366210936,
-                0.03338137435913086,
                 0.033416191101074216,
-                0.033442047119140624,
-                0.03320191955566406,
-                0.03322265625,
-                0.03342131042480469,
                 0.03347558212280274,
-                0.03333631896972656,
-                0.03323801422119141,
-                0.033255424499511715,
-                0.033296382904052735,
                 0.03349708938598633,
-                0.03344179153442383,
-                0.03343360137939453,
-                0.033358848571777344,
-                0.03338444900512695,
-                0.03333324813842774,
-                0.033342464447021485,
-                0.03333017730712891,
-                0.0334284782409668,
-                0.0334284782409668,
-                0.03329945755004883,
-                0.033271808624267575,
-                0.033173503875732424,
-                0.03332505416870117,
                 0.03348992156982422,
-                0.03328716659545899,
-                0.03332403182983398,
-                0.033219585418701174,
-                0.03335782241821289,
-                0.033339393615722655,
-                0.03322470474243164,
-                0.033423553466796874,
-                0.03352659225463867,
-                0.033291263580322264,
                 0.033478656768798826,
-                0.03340083312988281,
-                0.03338649749755859
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 30.036318245038174
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 993.992704,
             "max_vram": 4905.238528,
             "max_reserved": 4420.796416,
             "max_allocated": 4252.667392
         },
         "latency": {
             "unit": "s",
+            "mean": 0.04236017230392006,
+            "stdev": 0.000576452064979207,
             "values": [
+                0.04680534362792969,
+                0.044624897003173826,
+                0.04397568130493164,
+                0.04385996627807617,
+                0.04318003082275391,
+                0.04338175964355469,
+                0.04339507293701172,
+                0.04385279846191406,
+                0.04769792175292969,
+                0.044948768615722653,
+                0.042513664245605466,
+                0.04241305541992187,
+                0.04255667114257813,
+                0.04243046569824219,
+                0.042534912109375,
+                0.04235776138305664,
+                0.042175487518310545,
                 0.042379264831542966,
+                0.042262527465820314,
+                0.04238950347900391,
+                0.04217036819458008,
+                0.042380287170410154,
+                0.04207513427734375,
                 0.04237209701538086,
+                0.04212736129760742,
+                0.042380287170410154,
+                0.04197683334350586,
+                0.04235468673706055,
+                0.04194815826416016,
                 0.042365951538085936,
+                0.04203519821166992,
+                0.042359809875488284,
+                0.041987071990966796,
+                0.04235059356689453,
+                0.042103809356689455,
+                0.04236800003051758,
+                0.042103809356689455,
                 0.04237516784667969,
+                0.04207206344604492,
+                0.04238131332397461,
+                0.041987071990966796,
                 0.04235161590576172,
+                0.04207001495361328,
+                0.042395648956298826,
+                0.04211404800415039,
+                0.042372222900390624,
+                0.041882625579833986,
+                0.04239769744873047,
+                0.04194918441772461,
+                0.0423741455078125,
+                0.04214374542236328,
                 0.042382335662841795,
+                0.0420136947631836,
+                0.04239974212646484,
+                0.042028030395507815,
                 0.04236083221435547,
+                0.042024959564208986,
+                0.04237004852294922,
+                0.04215500640869141,
                 0.04237619018554688,
                 0.04203212738037109,
+                0.04236800003051758,
+                0.042533889770507816,
                 0.042393600463867184,
+                0.04199935913085937,
+                0.04236800003051758,
+                0.04215398406982422,
+                0.04238643264770508,
+                0.04209868621826172,
+                0.04238249588012695,
+                0.04219107055664063,
+                0.04239769744873047,
+                0.04200960159301758,
+                0.04240387344360352,
+                0.04211097717285156,
+                0.04236800003051758,
+                0.042090496063232424,
+                0.042393600463867184,
+                0.042033153533935545,
+                0.042363903045654294,
+                0.04212838363647461,
+                0.04240691375732422,
+                0.04217446517944336,
+                0.042365951538085936,
+                0.04201062393188477,
+                0.04236697769165039,
+                0.041954303741455076,
+                0.04237004852294922,
+                0.04208025741577148,
+                0.0423741455078125,
+                0.04204544067382812,
                 0.04235776138305664,
+                0.041935871124267575,
+                0.0424007682800293,
+                0.042060798645019534,
+                0.04239379119873047,
+                0.04198809432983398,
+                0.042635265350341796,
+                0.04254617691040039,
+                0.04236288070678711,
+                0.04196454238891602,
+                0.04237311935424805,
+                0.04196454238891602,
                 0.04236083221435547,
+                0.042068992614746094,
                 0.042385406494140625,
+                0.042019840240478515,
+                0.04236185455322266,
+                0.042159103393554685,
                 0.04237721633911133,
+                0.04225024032592774,
+                0.04239257431030274,
+                0.042054656982421876,
+                0.042401790618896484,
+                0.041973758697509765,
+                0.04236185455322266,
+                0.04198310470581055,
+                0.04237823867797851,
+                0.042243072509765625,
+                0.04244377517700195,
+                0.042178848266601565,
+                0.04235468673706055,
+                0.042261505126953126,
                 0.04235161590576172,
+                0.042229759216308595,
+                0.04241113662719727,
+                0.04233011245727539,
+                0.04236697769165039,
+                0.04229836654663086,
+                0.042352638244628905,
+                0.042090496063232424,
+                0.04239257431030274,
+                0.04217036819458008,
+                0.04238979339599609,
+                0.04210892868041992,
+                0.04236697769165039,
+                0.04209664154052734,
+                0.04236288070678711,
+                0.04214988708496094,
+                0.04236185455322266,
+                0.041957374572753905,
+                0.04234649658203125,
+                0.04209868621826172,
+                0.04237209701538086,
+                0.041957374572753905,
+                0.04237004852294922,
+                0.04242124938964844,
                 0.0423741455078125,
+                0.04202598571777344,
+                0.04237004852294922,
+                0.04204032135009766,
+                0.04237721633911133,
+                0.0421212158203125,
+                0.04237107086181641,
+                0.042052608489990234,
+                0.042363903045654294,
+                0.04207820892333984,
+                0.04239974212646484,
+                0.04204467010498047,
+                0.04238848114013672,
+                0.042057727813720705,
+                0.04237721633911133,
+                0.04198092651367188,
+                0.042382431030273435,
+                0.04211814498901367,
+                0.04235161590576172,
+                0.04206489562988281,
+                0.042385406494140625,
+                0.042191871643066405,
                 0.04240691375732422,
+                0.04207820892333984,
                 0.042363903045654294,
+                0.04212838363647461,
                 0.04238848114013672,
+                0.04204339218139649,
+                0.042363903045654294,
+                0.04208844757080078,
+                0.04237007904052734,
+                0.042233089447021484,
+                0.042385406494140625,
+                0.042155105590820315,
+                0.04237516784667969,
+                0.042241024017333983,
+                0.042369022369384765,
+                0.04214169692993164,
                 0.042379264831542966,
+                0.04200755310058594,
                 0.04237516784667969,
+                0.04221644973754883,
+                0.04237004852294922,
+                0.04222054290771484,
                 0.042382335662841795,
+                0.0421396484375,
                 0.04237619018554688,
+                0.042635265350341796,
+                0.04239052963256836,
+                0.04208332824707031,
+                0.042396671295166014,
+                0.04214169692993164,
+                0.042420223236083986,
+                0.04225024032592774,
+                0.042428417205810545,
+                0.042177536010742187,
+                0.04245328140258789,
+                0.042164222717285156,
+                0.04282572937011719,
+                0.042234878540039066,
+                0.04246527862548828,
+                0.04213555145263672,
+                0.04244172668457031,
+                0.042241024017333983,
+                0.042469375610351565,
+                0.042120193481445314,
+                0.04245708847045898,
+                0.041998336791992184,
+                0.04244070434570312,
+                0.04219100952148438,
+                0.04245811080932617,
+                0.042147838592529296,
+                0.04245401763916016,
+                0.04230144119262695,
+                0.042471424102783206,
+                0.04219903945922852,
+                0.042462368011474606,
+                0.04220723342895508,
+                0.04247859191894531,
+                0.04230144119262695,
+                0.042420223236083986,
+                0.0420136947631836,
+                0.0424510383605957,
+                0.042332256317138675,
+                0.04247347259521484,
+                0.042246177673339845,
+                0.04244172668457031,
+                0.04203007888793946,
+                0.04246131134033203,
+                0.04205158233642578
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 6043.4126226703165
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 993.992704,
             "max_vram": 5255.462912,
             "max_reserved": 4771.0208,
+            "max_allocated": 4715.61216
         },
         "latency": {
             "unit": "s",
+            "mean": 8.584606702804567,
+            "stdev": 0.012920709609988812,
             "values": [
+                8.597527412414557,
+                8.57168599319458
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 29.704331115913813
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.03366512432472383,
+            "stdev": 0.0008075856248906625,
             "values": [
+                0.03338943862915039,
+                0.03398678588867188,
+                0.03342214584350586,
+                0.033371105194091796,
+                0.03382067108154297,
+                0.03350143814086914,
+                0.03323878479003906,
+                0.03339571380615235,
+                0.033360126495361325,
+                0.0334273910522461,
+                0.03330131149291992,
+                0.033317886352539065,
+                0.033451103210449216,
+                0.033478561401367186,
+                0.03342335891723633,
+                0.03355750274658203,
+                0.033438720703125,
+                0.033372417449951175,
+                0.03343027114868164,
+                0.03374694442749023,
+                0.033312767028808594,
+                0.03350527954101563,
+                0.03364556884765625,
                 0.033463294982910154,
+                0.0334284782409668,
+                0.033459201812744144,
+                0.033420318603515624,
+                0.03340284729003906,
+                0.03374387359619141,
+                0.03347353744506836,
+                0.03338240051269531,
+                0.033462272644042966,
+                0.033318145751953125,
+                0.03343334579467774,
+                0.03338137435913086,
+                0.03347967910766601,
+                0.03346944046020508,
+                0.03356364822387695,
+                0.03350630569458008,
+                0.03352268981933594,
+                0.03346764755249024,
+                0.03343657684326172,
+                0.03349513626098633,
+                0.03339750289916992,
                 0.033438720703125,
+                0.033416191101074216,
                 0.033358848571777344,
+                0.033716224670410154,
+                0.03358745574951172,
+                0.033355552673339846,
+                0.033523681640625,
+                0.03346044921875,
+                0.03342115020751953,
+                0.0334345588684082,
+                0.03344179153442383,
                 0.033363967895507815,
+                0.033345535278320314,
+                0.03341209411621094,
+                0.0335175666809082,
+                0.033519615173339845,
+                0.03346124649047852,
+                0.03347257614135742,
+                0.0334345588684082,
+                0.033519615173339845,
+                0.03408281707763672,
+                0.03343667221069336,
+                0.033428543090820315,
+                0.033484737396240236,
+                0.03342544174194336,
+                0.03349615859985351,
+                0.03346233749389648,
+                0.03336070251464844,
+                0.0333383674621582,
+                0.033298431396484376,
+                0.033393665313720705,
+                0.033503231048583985,
+                0.0334571533203125,
+                0.03333222579956055,
+                0.033432575225830076,
+                0.03372355270385742,
+                0.033423423767089844,
+                0.03358185577392578,
+                0.033446910858154294,
+                0.033484798431396484,
+                0.03343769454956055,
+                0.03339775848388672,
+                0.033392799377441405,
+                0.03349795150756836,
+                0.03346022415161133,
+                0.03348889541625977,
+                0.03346944046020508,
+                0.03344076919555664,
+                0.033448959350585936,
+                0.033495040893554685,
+                0.033500160217285156,
+                0.03350425720214844,
+                0.033506431579589845,
+                0.03343859100341797,
                 0.033329151153564454,
+                0.033448959350585936,
+                0.033617919921875,
+                0.033460289001464846,
+                0.03367734527587891,
+                0.03338947296142578,
+                0.03341641616821289,
+                0.03336272048950195,
                 0.03341209411621094,
+                0.03331916809082031,
+                0.03346918487548828,
+                0.03339497756958008,
+                0.033419998168945315,
                 0.03339571380615235,
+                0.03351859283447266,
+                0.0334246711730957,
+                0.033495006561279295,
+                0.033521408081054686,
+                0.03341315078735352,
+                0.03335369491577148,
+                0.03343667221069336,
+                0.0332492790222168,
+                0.033269790649414065,
+                0.033390560150146485,
+                0.03363020706176758,
+                0.03348992156982422,
+                0.033484798431396484,
+                0.03378995132446289,
+                0.03370614242553711,
+                0.03338838577270508,
+                0.033539070129394534,
+                0.033501182556152344,
+                0.033432575225830076,
+                0.033377281188964845,
+                0.03328307342529297,
+                0.033364990234375,
+                0.03348787307739258,
+                0.03339775848388672,
+                0.033377281188964845,
+                0.03345305633544922,
+                0.033361183166503904,
+                0.033774303436279296,
+                0.033511425018310545,
+                0.033309696197509765,
+                0.03349734497070313,
+                0.03345280075073242,
+                0.0335206413269043,
+                0.033391616821289063,
+                0.033516544342041016,
+                0.03326464080810547,
+                0.033600513458251956,
+                0.03341107177734375,
+                0.033438720703125,
                 0.033312767028808594,
+                0.03356979370117188,
+                0.03354316711425781,
+                0.03347251129150391,
+                0.03335782241821289,
+                0.03340492630004883,
+                0.03364659118652344,
+                0.03351555252075195,
+                0.033349601745605466,
+                0.033347583770751955,
+                0.033274974822998044,
+                0.03369257736206055,
+                0.03335497665405274,
+                0.03340367889404297,
                 0.0334284782409668,
+                0.03346022415161133,
+                0.03338240051269531,
+                0.033492992401123044,
+                0.03343667221069336,
+                0.03348787307739258,
+                0.033363967895507815,
+                0.03360995101928711,
+                0.03344771194458008,
+                0.03347251129150391,
+                0.033495040893554685,
+                0.03343974304199219,
+                0.033492992401123044,
+                0.03348582458496094,
+                0.033525920867919924,
+                0.03347747039794922,
+                0.03346432113647461,
+                0.03349401473999023,
+                0.033481727600097655,
+                0.03358003234863281,
+                0.033788928985595705,
+                0.03366604614257813,
+                0.03353702545166016,
+                0.03352166366577149,
+                0.03353190231323242,
+                0.033598464965820314,
+                0.03341823959350586,
+                0.03353926467895508,
+                0.033567550659179685,
+                0.03352691268920899,
+                0.036181888580322265,
+                0.03810201644897461,
+                0.03954995346069336,
+                0.03883212661743164,
+                0.03791155242919922,
+                0.03795558547973633,
+                0.03790950393676758,
+                0.03722768020629883,
+                0.038012767791748045,
+                0.037926910400390625,
+                0.0352174072265625,
+                0.034977054595947264,
+                0.0352918701171875,
+                0.036751361846923826,
+                0.03619942474365234,
+                0.03588198471069336,
+                0.0339947509765625,
+                0.0335994873046875,
+                0.03356364822387695,
+                0.03345817565917969,
+                0.03349935913085938,
+                0.03351865768432617,
+                0.03349094390869141,
+                0.03415625762939453,
+                0.033898494720458985,
+                0.03359539031982422,
+                0.033549312591552735,
+                0.033501182556152344,
+                0.03368268966674805,
+                0.03379276657104492,
+                0.03378995132446289,
+                0.033514495849609374,
+                0.033432575225830076,
+                0.03348384094238281,
+                0.03355539321899414,
+                0.03356675338745117,
+                0.03345916748046875,
+                0.03362201690673828,
+                0.03345532989501953,
+                0.03349817657470703,
+                0.0334681282043457,
+                0.03339775848388672,
+                0.033633281707763675,
+                0.033600513458251956,
+                0.033582080841064454,
+                0.03341926574707031,
+                0.033416191101074216,
+                0.033545246124267576,
+                0.03388639831542969,
+                0.03340163040161133,
                 0.03341107177734375,
+                0.03357519912719727,
+                0.03354185485839844,
                 0.033430526733398434,
+                0.033565696716308595,
+                0.033508350372314456,
+                0.03342745590209961,
+                0.03358310317993164,
+                0.033582080841064454,
+                0.03366400146484375,
+                0.033670398712158205,
+                0.033971904754638675,
+                0.03348889541625977,
+                0.03351244735717773,
+                0.03350732803344727,
+                0.03363225555419922,
+                0.03361382293701172,
+                0.033568767547607424,
+                0.033567745208740236,
+                0.033495040893554685,
+                0.03340902328491211,
+                0.033430526733398434,
+                0.033413120269775394,
+                0.03338143920898438,
+                0.03348297500610352,
+                0.03358998489379883,
+                0.03361894226074219,
+                0.0335810546875,
+                0.033641792297363284,
+                0.03351827239990234,
+                0.033535999298095705,
+                0.033552383422851564,
+                0.03355033493041992,
+                0.03381043243408203,
+                0.03360768127441406,
+                0.033576961517333984,
+                0.03360768127441406,
+                0.033691646575927735,
+                0.033600513458251956,
+                0.03356159973144531,
+                0.03351859283447266,
+                0.03352067184448242,
+                0.033581024169921876,
+                0.03356364822387695,
+                0.03353702545166016,
+                0.03359561538696289,
+                0.033702686309814454,
+                0.03357299041748047,
+                0.033516414642333986,
+                0.033562625885009766,
+                0.03350527954101563,
+                0.03343564987182617,
+                0.034904064178466795,
+                0.040970241546630856,
+                0.03864390563964844,
+                0.03719148635864258,
+                0.03498905563354492,
+                0.03358924865722656,
+                0.03352371215820313,
+                0.03365478515625,
+                0.033597728729248044,
+                0.033610591888427736,
+                0.0341635856628418,
+                0.03346944046020508,
+                0.03350527954101563,
+                0.033552383422851564,
                 0.033416191101074216,
+                0.03345100784301758,
+                0.03351347351074219,
+                0.033614849090576174,
+                0.03357593536376953,
+                0.03394355010986328,
                 0.03347558212280274,
+                0.03351347351074219,
+                0.033519615173339845,
+                0.03357798385620117,
+                0.03370598220825195,
+                0.03351039886474609,
+                0.03357798385620117,
+                0.03352576065063476,
                 0.03349708938598633,
+                0.03343564987182617,
+                0.03348598480224609,
+                0.03346518325805664,
+                0.033685504913330076,
+                0.033486846923828126,
+                0.03360358428955078,
+                0.033529857635498046,
+                0.033616897583007815,
+                0.03362713623046875,
+                0.03355955123901367,
+                0.033549312591552735,
+                0.03355340957641602,
+                0.03346432113647461,
+                0.03357183837890625,
+                0.03360870361328125,
+                0.033805313110351565,
+                0.0337151985168457,
+                0.03364659118652344,
+                0.03357593536376953,
+                0.033800193786621094,
+                0.03346636962890625,
+                0.03344704055786133,
+                0.033489791870117185,
+                0.03355267333984375,
+                0.033667808532714845,
+                0.033565696716308595,
+                0.03355750274658203,
+                0.033560577392578124,
+                0.033492992401123044,
+                0.03346233749389648,
+                0.0334796142578125,
+                0.03374182510375977,
+                0.033655807495117186,
+                0.03349708938598633,
+                0.03366604614257813,
+                0.03388518524169922,
+                0.0335093765258789,
+                0.033519615173339845,
+                0.0337367057800293,
+                0.03350342559814453,
+                0.0334477424621582,
+                0.033532928466796875,
+                0.033503231048583985,
+                0.033568767547607424,
+                0.033540287017822266,
+                0.03367929458618164,
+                0.03362086486816406,
+                0.03359334564208984,
+                0.033565696716308595,
+                0.03343462371826172,
+                0.03383603286743164,
+                0.033587200164794925,
+                0.03348889541625977,
+                0.03375718307495117,
+                0.033414142608642575,
+                0.033588222503662106,
+                0.03347763061523437,
+                0.03347763061523437,
+                0.033337345123291014,
+                0.033503231048583985,
+                0.03336294555664063,
+                0.033465343475341795,
+                0.03362918472290039,
+                0.03353190231323242,
+                0.03345305633544922,
+                0.03349094390869141,
+                0.033516544342041016,
+                0.033514751434326175,
+                0.03344367980957031,
+                0.033514400482177735,
+                0.033461311340332034,
+                0.03348678588867188,
+                0.03369472122192383,
+                0.0335247688293457,
+                0.03361276626586914,
+                0.033328128814697267,
+                0.03353497695922852,
+                0.03355955123901367,
+                0.033859809875488284,
+                0.03348867034912109,
+                0.033459201812744144,
+                0.033452030181884765,
+                0.033474624633789064,
+                0.03349113464355469,
+                0.033398529052734376,
+                0.03338137435913086,
+                0.0337448959350586,
+                0.033459201812744144,
+                0.03349401473999023,
+                0.03330691146850586,
+                0.033372161865234375,
+                0.03338723373413086,
+                0.033410049438476565,
+                0.03351347351074219,
+                0.033576961517333984,
+                0.03346944046020508,
+                0.033807361602783206,
+                0.033500160217285156,
+                0.033498111724853515,
+                0.03347382354736328,
+                0.03347017669677734,
+                0.03367222213745117,
+                0.033472480773925783,
+                0.03373056030273437,
+                0.033329151153564454,
+                0.03344998550415039,
+                0.033372161865234375,
+                0.03342233657836914,
+                0.033447071075439455,
+                0.03367308807373047,
+                0.03379606246948242,
+                0.03354111862182617,
+                0.03330867385864258,
+                0.033406238555908206,
+                0.033344223022460935,
+                0.03347251129150391,
+                0.03360255813598633,
+                0.03360255813598633,
+                0.03338854217529297,
+                0.03337343978881836,
+                0.03332992172241211,
+                0.03338137435913086,
+                0.033442817687988284,
+                0.033516544342041016,
                 0.03348992156982422,
+                0.03335168075561523,
+                0.0334317741394043,
+                0.033481502532958986,
+                0.03336294555664063,
+                0.03345510482788086,
+                0.033649665832519535,
+                0.033530879974365234,
+                0.033484798431396484,
+                0.033414142608642575,
+                0.033345535278320314,
+                0.03343564987182617,
+                0.03360563278198242,
+                0.03374195098876953,
+                0.033631103515625,
+                0.033495040893554685,
+                0.033393665313720705,
+                0.03339775848388672,
+                0.03355033493041992,
+                0.03340390396118164,
+                0.03345305633544922,
+                0.0335175666809082,
+                0.033396961212158204,
+                0.03345078277587891,
+                0.033524734497070316,
+                0.033546398162841794,
+                0.03352479934692383,
+                0.03351631927490235,
+                0.03348992156982422,
+                0.03357491302490234,
+                0.033463294982910154,
+                0.03350630569458008,
+                0.033860607147216795,
+                0.0334284782409668,
+                0.03344815826416016,
+                0.033487648010253904,
+                0.033600513458251956,
+                0.0334951057434082,
+                0.03370284652709961,
+                0.03373382568359375,
+                0.033530689239501955,
                 0.033478656768798826,
+                0.03353702545166016,
+                0.03354111862182617,
+                0.033486846923828126,
+                0.03347455978393555,
+                0.03395481491088867,
+                0.03349094390869141,
+                0.033721343994140625,
+                0.033454078674316406,
+                0.03344076919555664,
+                0.033503231048583985,
+                0.0336363525390625,
+                0.03350630569458008,
+                0.03344384002685547,
+                0.033521854400634765,
+                0.033528640747070314,
+                0.0335206413269043,
+                0.03358924865722656,
+                0.03347267150878906
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 29.70433111591378
         },
         "energy": null,
         "efficiency": null