kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Mar 14

Commit

85b4f7c

•

1 Parent(s): b0fef22

Upload /Mistral-7B-v0.1/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Mistral-7B-v0.1/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +764 -763

Mistral-7B-v0.1/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,299 +2,300 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 985.341952,
             "max_vram": 15755.902976,
             "max_reserved": 15271.460864,
             "max_allocated": 15114.469888
         },
         "latency": {
             "unit": "s",
-            "mean": 0.035987248907843924,
-            "stdev": 0.00021574759418921462,
             "values": [
-                0.03843862533569336,
-                0.03772723388671875,
-                0.03614720153808594,
-                0.036101119995117184,
                 0.036176895141601564,
-                0.03613798522949219,
-                0.03643904113769531,
-                0.03620249557495117,
-                0.036176895141601564,
-                0.03693772888183594,
-                0.03680460739135742,
-                0.036363552093505856,
-                0.03591680145263672,
-                0.03592192077636719,
-                0.0358922233581543,
-                0.03592192077636719,
-                0.03593318557739258,
-                0.035937278747558594,
-                0.035958782196044925,
-                0.035972095489501955,
-                0.03597312164306641,
-                0.035866622924804685,
-                0.03590569686889648,
-                0.03593011093139648,
-                0.03598233413696289,
-                0.03594854354858398,
-                0.035937278747558594,
-                0.03593625640869141,
-                0.03592806243896484,
-                0.035958782196044925,
-                0.035952640533447267,
-                0.0359444465637207,
-                0.035888126373291016,
                 0.03601203155517578,
-                0.0359024658203125,
-                0.03589017486572266,
-                0.035963905334472655,
-                0.03593523025512695,
-                0.035947521209716796,
-                0.036975616455078124,
-                0.0359813117980957,
-                0.035945472717285154,
-                0.03592806243896484,
-                0.03592192077636719,
                 0.035929088592529294,
-                0.03596083068847656,
-                0.03594035339355469,
-                0.035955745697021486,
-                0.03589120101928711,
-                0.03592499160766602,
-                0.03595673751831055,
-                0.0359106559753418,
-                0.035980289459228515,
-                0.035983360290527344,
                 0.03590553665161133,
-                0.03590963363647461,
-                0.035942401885986325,
-                0.035931137084960936,
-                0.03593952178955078,
-                0.03594035339355469,
-                0.0359659538269043,
-                0.035942401885986325,
-                0.03612499237060547,
-                0.03594956970214844,
-                0.035939327239990236,
-                0.03592806243896484,
-                0.035958782196044925,
-                0.03593625640869141,
-                0.03592704010009766,
-                0.03592499160766602,
-                0.03597824096679687,
-                0.035952640533447267,
                 0.035929088592529294,
-                0.03593625640869141,
-                0.03595775985717774,
                 0.03603558349609375,
-                0.03607551956176758,
-                0.035953662872314454,
-                0.03595673751831055,
-                0.03597625732421875,
-                0.035922943115234376,
-                0.0359628791809082,
-                0.03595161437988281,
-                0.036015392303466794,
-                0.03597824096679687,
-                0.03595161437988281,
-                0.035929088592529294,
-                0.035969024658203126,
-                0.03593011093139648,
-                0.03597619247436523,
-                0.035963905334472655,
-                0.03594137573242188,
-                0.03594649505615234,
-                0.035999744415283204,
-                0.03595775985717774,
-                0.03597619247436523,
-                0.035958782196044925,
-                0.03592416000366211,
-                0.035912704467773435,
-                0.035920894622802735,
-                0.03590860748291016,
-                0.035969024658203126,
-                0.03592704010009766,
-                0.03602329635620117,
-                0.03599359893798828,
-                0.0359251823425293,
-                0.03596492767333984,
-                0.03601408004760742,
-                0.035970046997070314,
-                0.03595596694946289,
-                0.035937278747558594,
-                0.035945472717285154,
-                0.03594137573242188,
-                0.035932159423828124,
-                0.035963905334472655,
-                0.03594137573242188,
-                0.03599257659912109,
-                0.035966976165771485,
-                0.03612364959716797,
-                0.03595161437988281,
-                0.035945472717285154,
-                0.03600281524658203,
                 0.03595775985717774,
-                0.035963905334472655,
-                0.03594035339355469,
-                0.035950721740722655,
-                0.03597926330566406,
-                0.03600896072387695,
-                0.035942401885986325,
-                0.03597926330566406,
-                0.035963905334472655,
-                0.03598233413696289,
-                0.03606425476074219,
-                0.03592704010009766,
-                0.03594460678100586,
-                0.03594137573242188,
-                0.035952640533447267,
-                0.03596291351318359,
-                0.035922943115234376,
-                0.03592601776123047,
-                0.03592396926879883,
-                0.03594137573242188,
-                0.03594342422485351,
-                0.035929088592529294,
-                0.0358922233581543,
-                0.036050945281982424,
-                0.03590553665161133,
-                0.035922943115234376,
-                0.03592396926879883,
-                0.035915775299072264,
-                0.035912704467773435,
-                0.03593625640869141,
-                0.03595372772216797,
-                0.03594956970214844,
-                0.03599155044555664,
-                0.03593523025512695,
-                0.03589529418945313,
-                0.035915775299072264,
                 0.03595161437988281,
-                0.03598144149780273,
-                0.0359106559753418,
-                0.03590348815917969,
-                0.03593318557739258,
-                0.035980289459228515,
-                0.035901439666748046,
-                0.035939327239990236,
-                0.0359106559753418,
-                0.035953662872314454,
-                0.03592601776123047,
-                0.035970046997070314,
-                0.03588915252685547,
-                0.035896320343017575,
-                0.035904510498046875,
-                0.03593011093139648,
-                0.03606016159057617,
-                0.03591088104248047,
-                0.03589427185058594,
-                0.0359628791809082,
-                0.035934207916259765,
-                0.03592601776123047,
-                0.03591705703735352,
-                0.0359466552734375,
-                0.035884033203125,
-                0.03591987228393555,
-                0.035969024658203126,
-                0.035896320343017575,
                 0.03590963363647461,
-                0.03588710403442383,
-                0.036071422576904294,
-                0.035915775299072264,
-                0.03594956970214844,
-                0.035914752960205076,
-                0.035915775299072264,
-                0.03592806243896484,
-                0.03593830490112305,
-                0.035901439666748046,
-                0.03592396926879883,
-                0.03593625640869141,
-                0.035904510498046875,
                 0.03586969757080078,
                 0.035883007049560545,
-                0.035988479614257815,
-                0.03588608169555664,
-                0.03592396926879883,
-                0.03595161437988281,
-                0.03593625640869141,
-                0.035975166320800785,
-                0.03593318557739258,
-                0.035899391174316404,
-                0.03592499160766602,
-                0.0359106559753418,
-                0.03592396926879883,
-                0.03593830490112305,
-                0.03595673751831055,
                 0.035942401885986325,
-                0.03592806243896484,
-                0.03592499160766602,
-                0.0359598388671875,
-                0.035901439666748046,
-                0.03594854354858398,
                 0.035947521209716796,
-                0.03593011093139648,
-                0.035915775299072264,
-                0.035942401885986325,
-                0.035901439666748046,
-                0.0358922233581543,
-                0.03592704010009766,
-                0.035931137084960936,
-                0.03590758514404297,
-                0.03590655899047852,
-                0.03608473587036133,
-                0.035961856842041014,
-                0.03594140625,
-                0.03595980834960937,
-                0.035934207916259765,
-                0.035955711364746096,
-                0.03592704010009766,
-                0.03594137573242188,
-                0.036029441833496094,
-                0.03604275131225586,
-                0.036001792907714845,
-                0.03600076675415039,
                 0.036055038452148434,
-                0.0360447998046875,
-                0.03612979125976563,
-                0.036038654327392575,
-                0.035999744415283204,
-                0.03601715087890625,
-                0.03602022552490235,
-                0.036050945281982424,
-                0.035975166320800785,
-                0.03603968048095703,
-                0.03599792098999023,
-                0.036021247863769534,
-                0.03602841567993164,
-                0.035974334716796875,
-                0.035945472717285154,
-                0.03602329635620117,
-                0.03597747039794922,
-                0.03599052810668945,
-                0.03600815963745117,
-                0.035999744415283204,
-                0.036103168487548826,
-                0.03600691223144531,
-                0.03599462509155273,
-                0.03599462509155273,
-                0.03597619247436523,
-                0.03599462509155273,
-                0.03597107315063477,
-                0.03599462509155273,
-                0.03599155044555664,
-                0.03597312164306641,
-                0.036001792907714845,
-                0.03600080108642578,
-                0.0360079345703125,
-                0.03595468902587891,
-                0.03594956970214844,
-                0.0359628791809082
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 7113.630737808391
         },
         "energy": null,
         "efficiency": null
@@ -302,23 +303,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 985.341952,
-            "max_vram": 15923.675136,
-            "max_reserved": 15439.233024,
             "max_allocated": 15238.453248
         },
         "latency": {
             "unit": "s",
-            "mean": 7.1229505767822285,
-            "stdev": 0.00452053070068148,
             "values": [
-                7.12747110748291,
-                7.118430046081547
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.79977107116128
         },
         "energy": null,
         "efficiency": null
@@ -327,524 +328,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.027933139516793035,
-            "stdev": 0.00013863113423522085,
             "values": [
-                0.02992742347717285,
-                0.029331775665283204,
-                0.02788319969177246,
-                0.027845632553100585,
-                0.027849727630615235,
-                0.027805696487426756,
-                0.027852800369262694,
-                0.027850751876831056,
-                0.027831296920776367,
-                0.027850751876831056,
-                0.027958431243896485,
-                0.02785161590576172,
-                0.0278569278717041,
-                0.027859935760498045,
-                0.02779648017883301,
-                0.02790297508239746,
-                0.027836416244506838,
-                0.027822080612182616,
-                0.027811840057373048,
-                0.027823104858398437,
-                0.027835391998291017,
-                0.027995328903198242,
-                0.027817792892456054,
-                0.027900928497314452,
-                0.02787046432495117,
-                0.02780953598022461,
-                0.027860992431640624,
-                0.028075008392333983,
-                0.027883520126342775,
-                0.027865087509155274,
-                0.027867136001586915,
-                0.027848703384399414,
-                0.027900928497314452,
-                0.027790336608886718,
-                0.0279418888092041,
-                0.027858943939208985,
-                0.02785513687133789,
-                0.02784227180480957,
-                0.02786623954772949,
-                0.02795916748046875,
-                0.027914304733276368,
-                0.02787424087524414,
-                0.027900928497314452,
-                0.027838464736938476,
-                0.02788476753234863,
-                0.02789455986022949,
-                0.027903200149536133,
-                0.02792835235595703,
-                0.027928607940673828,
-                0.027907039642333983,
-                0.027910144805908203,
-                0.02795929527282715,
-                0.027897855758666993,
-                0.02790297508239746,
-                0.027855871200561523,
-                0.027850751876831056,
-                0.027824127197265625,
-                0.02788761520385742,
-                0.027913215637207032,
-                0.027872447967529298,
-                0.02782598304748535,
-                0.027838464736938476,
-                0.02785206413269043,
-                0.02787708854675293,
-                0.027843584060668947,
-                0.027884544372558592,
-                0.027873279571533204,
-                0.02791116714477539,
-                0.027892736434936522,
-                0.02793471908569336,
                 0.02794495964050293,
-                0.028044288635253906,
-                0.027882816314697266,
-                0.02790060806274414,
-                0.02790809631347656,
-                0.027889663696289063,
-                0.027888639450073242,
-                0.027866111755371094,
-                0.02791116714477539,
-                0.02797875213623047,
-                0.027861087799072266,
-                0.02795817565917969,
-                0.02812313652038574,
-                0.027864063262939453,
-                0.02778828811645508,
-                0.0286167049407959,
-                0.028886016845703126,
-                0.028100608825683594,
-                0.027857952117919922,
-                0.027945951461791994,
-                0.027838720321655273,
-                0.027878143310546874,
-                0.02792755126953125,
-                0.027923456192016603,
-                0.02797260856628418,
-                0.02795212745666504,
-                0.02794086456298828,
-                0.02792959976196289,
-                0.027882816314697266,
-                0.02810643196105957,
-                0.027881696701049806,
-                0.027904800415039063,
-                0.02787843132019043,
-                0.027882463455200197,
-                0.02795315170288086,
                 0.027926528930664062,
-                0.027879615783691407,
-                0.027866943359375,
-                0.027881471633911133,
-                0.027865087509155274,
-                0.02788380813598633,
-                0.02785456085205078,
-                0.027880640029907228,
-                0.027861824035644533,
-                0.027875328063964845,
-                0.027867136001586915,
-                0.02792448043823242,
-                0.027880447387695313,
-                0.02791116714477539,
-                0.02793164825439453,
-                0.02794803237915039,
-                0.027913215637207032,
-                0.02795827293395996,
-                0.027869184494018553,
-                0.027904319763183593,
-                0.02791289520263672,
-                0.02793267250061035,
-                0.027873279571533204,
-                0.027849727630615235,
-                0.02793471908569336,
                 0.027868160247802733,
-                0.027874303817749024,
-                0.02790505599975586,
-                0.027891679763793944,
-                0.02789580726623535,
-                0.02791116714477539,
-                0.027889663696289063,
-                0.02790809631347656,
-                0.02789580726623535,
-                0.027913215637207032,
-                0.0279418888092041,
-                0.027975679397583008,
-                0.02792140769958496,
-                0.0279552001953125,
-                0.027906208038330077,
-                0.027929439544677734,
-                0.02793984031677246,
-                0.027905023574829102,
-                0.027898015975952147,
-                0.027917152404785157,
-                0.02791935920715332,
-                0.02793267250061035,
-                0.027915264129638673,
-                0.02795622444152832,
-                0.028094463348388672,
-                0.02792550468444824,
-                0.02794905662536621,
-                0.027963392257690428,
-                0.028026016235351562,
-                0.027937631607055664,
-                0.02791731262207031,
-                0.027987968444824218,
-                0.027918336868286132,
-                0.027930624008178712,
-                0.02795315170288086,
-                0.02792448043823242,
-                0.027892736434936522,
-                0.027936767578125,
                 0.02792755126953125,
-                0.02792959976196289,
-                0.02797260856628418,
-                0.028213247299194336,
-                0.027881568908691406,
-                0.027962272644042968,
                 0.027909120559692382,
-                0.02794598388671875,
-                0.02795212745666504,
-                0.027910144805908203,
-                0.0279685115814209,
-                0.02792857551574707,
-                0.02798489570617676,
-                0.02794086456298828,
-                0.027979776382446288,
-                0.02792448043823242,
-                0.027992063522338868,
-                0.02791219139099121,
-                0.027988191604614257,
-                0.02792732810974121,
-                0.027991071701049804,
-                0.027942880630493164,
-                0.027944992065429688,
-                0.02795516777038574,
-                0.02799001693725586,
-                0.02798182487487793,
-                0.02795008087158203,
-                0.0279736328125,
-                0.027910144805908203,
-                0.02794803237915039,
-                0.027998207092285156,
-                0.027982847213745117,
-                0.02796031951904297,
-                0.027992063522338868,
-                0.027930624008178712,
-                0.02797260856628418,
-                0.02799843215942383,
-                0.027974431991577148,
-                0.027931743621826172,
-                0.0279837760925293,
-                0.027997215270996093,
-                0.02807907295227051,
-                0.028030975341796875,
-                0.027982847213745117,
-                0.02796544075012207,
-                0.027987968444824218,
-                0.028022783279418945,
-                0.02795827293395996,
-                0.028037120819091797,
-                0.027997184753417968,
-                0.028047359466552735,
-                0.027999231338500977,
-                0.028000255584716797,
-                0.02799308776855469,
-                0.02797670364379883,
-                0.028011520385742186,
-                0.028018688201904295,
-                0.027982847213745117,
-                0.02818079948425293,
-                0.028029632568359376,
-                0.027998207092285156,
-                0.028014591217041016,
-                0.02800230407714844,
-                0.02800230407714844,
-                0.028045312881469726,
-                0.028025856018066408,
-                0.028013568878173828,
-                0.027995136260986327,
-                0.028000255584716797,
-                0.028000255584716797,
-                0.02795827293395996,
-                0.027966463088989257,
-                0.028037439346313475,
-                0.027996864318847656,
-                0.028018688201904295,
-                0.028227584838867188,
-                0.028049407958984376,
-                0.028036096572875976,
-                0.027994176864624024,
-                0.027978687286376952,
-                0.02794393539428711,
-                0.028012544631958007,
-                0.028040191650390626,
-                0.02795724868774414,
-                0.027936767578125,
-                0.02795622444152832,
-                0.02794291114807129,
-                0.0278822078704834,
-                0.027841535568237305,
-                0.027809823989868164,
-                0.02782716751098633,
-                0.02785001564025879,
-                0.0278504638671875,
-                0.027863040924072265,
-                0.027849727630615235,
-                0.027792383193969726,
-                0.027856895446777344,
-                0.027827199935913087,
-                0.027827199935913087,
-                0.027794624328613283,
-                0.027830080032348634,
-                0.027824127197265625,
                 0.027859968185424806,
-                0.027836416244506838,
-                0.027894880294799803,
-                0.027906240463256834,
-                0.027821887969970704,
-                0.027813791275024414,
-                0.027883520126342775,
-                0.02792550468444824,
-                0.02776678466796875,
-                0.0278703670501709,
-                0.02787004852294922,
-                0.027865087509155274,
-                0.027892736434936522,
-                0.02784390449523926,
-                0.027822784423828125,
-                0.027849727630615235,
                 0.027910144805908203,
-                0.027849760055541992,
-                0.027846624374389648,
                 0.027857919692993165,
                 0.027869184494018553,
-                0.02785408020019531,
-                0.027915008544921877,
-                0.027837440490722655,
-                0.027782144546508788,
-                0.027819007873535157,
-                0.027820032119750978,
-                0.027991039276123047,
-                0.027807743072509765,
-                0.027923583984375,
-                0.027935615539550783,
                 0.02791628837585449,
-                0.027881471633911133,
-                0.027856096267700196,
-                0.02788534355163574,
-                0.027859136581420897,
-                0.02787308883666992,
-                0.027873279571533204,
-                0.027820032119750978,
                 0.02780467224121094,
-                0.027869184494018553,
-                0.027883712768554687,
-                0.02786105537414551,
-                0.027901695251464843,
-                0.028049407958984376,
                 0.027893760681152343,
-                0.027838464736938476,
-                0.027864063262939453,
                 0.02788761520385742,
-                0.027801599502563477,
-                0.02791219139099121,
-                0.027893760681152343,
-                0.027882495880126954,
-                0.027837440490722655,
-                0.027824127197265625,
                 0.027892736434936522,
                 0.027833343505859375,
                 0.027897855758666993,
-                0.027885568618774413,
-                0.02794905662536621,
-                0.02791731262207031,
-                0.02790297508239746,
-                0.027872255325317383,
                 0.02790399932861328,
-                0.028000255584716797,
-                0.027893760681152343,
-                0.028014591217041016,
                 0.027886592864990234,
-                0.027891712188720705,
-                0.02779648017883301,
-                0.027890687942504884,
-                0.02779363250732422,
-                0.027860767364501954,
-                0.027867359161376955,
-                0.027907871246337892,
-                0.028010496139526365,
-                0.027847679138183593,
-                0.027883615493774414,
-                0.02791926383972168,
-                0.02791219139099121,
-                0.02793369674682617,
                 0.027888639450073242,
-                0.027982847213745117,
-                0.027886592864990234,
-                0.027938816070556642,
-                0.027819007873535157,
-                0.02790399932861328,
-                0.027875328063964845,
-                0.02793574333190918,
-                0.027898015975952147,
-                0.02782908821105957,
-                0.027868160247802733,
-                0.027852800369262694,
-                0.0278702392578125,
-                0.02787222480773926,
-                0.027911327362060548,
-                0.02783420753479004,
-                0.02796441650390625,
-                0.027878400802612304,
-                0.028066816329956053,
-                0.027840511322021484,
                 0.027883520126342775,
-                0.02790297508239746,
-                0.027821311950683592,
-                0.027857664108276368,
-                0.027849760055541992,
-                0.027896799087524415,
-                0.027869184494018553,
-                0.02792959976196289,
-                0.02794086456298828,
-                0.02791219139099121,
-                0.028007423400878906,
-                0.02792959976196289,
-                0.027900224685668946,
-                0.027863744735717774,
-                0.027909120559692382,
-                0.0279552001953125,
-                0.027983871459960938,
-                0.027842559814453126,
-                0.027886592864990234,
-                0.027918336868286132,
                 0.027851776123046876,
-                0.02795315170288086,
-                0.027909120559692382,
-                0.02793779182434082,
-                0.02790399932861328,
-                0.027915264129638673,
-                0.027867136001586915,
-                0.027894784927368164,
-                0.027889888763427736,
-                0.027906848907470704,
-                0.027909120559692382,
-                0.027936767578125,
                 0.027839487075805663,
-                0.027922431945800782,
-                0.0279418888092041,
-                0.02793267250061035,
                 0.027851776123046876,
-                0.027923456192016603,
-                0.02790297508239746,
-                0.027897855758666993,
-                0.027876575469970702,
-                0.027903776168823242,
-                0.027872447967529298,
-                0.027949888229370116,
-                0.027848928451538087,
-                0.028020511627197264,
-                0.027884544372558592,
-                0.02792550468444824,
-                0.02789580726623535,
-                0.027923456192016603,
-                0.02795417594909668,
-                0.027884544372558592,
-                0.02793164825439453,
-                0.02790809631347656,
-                0.027940128326416017,
-                0.02785558319091797,
-                0.027998207092285156,
-                0.02795417594909668,
-                0.0279418888092041,
-                0.027876352310180662,
-                0.02791731262207031,
                 0.027889663696289063,
-                0.02787942314147949,
-                0.027854848861694335,
-                0.027897855758666993,
-                0.02787942314147949,
-                0.02789990425109863,
-                0.027892736434936522,
-                0.02793164825439453,
-                0.027906047821044923,
-                0.028054527282714844,
-                0.028037120819091797,
-                0.02792959976196289,
-                0.02799411201477051,
-                0.028045440673828127,
-                0.02797657585144043,
-                0.028091615676879882,
-                0.0279467830657959,
-                0.028004415512084962,
-                0.027964351654052734,
-                0.02795827293395996,
-                0.02792959976196289,
-                0.027910144805908203,
-                0.027900928497314452,
-                0.027920671463012695,
-                0.027931360244750975,
-                0.028000255584716797,
-                0.027963392257690428,
-                0.02811801528930664,
-                0.02795724868774414,
-                0.02793471908569336,
-                0.028017663955688478,
-                0.027987968444824218,
-                0.02796544075012207,
                 0.028044288635253906,
                 0.028071935653686524,
-                0.028028032302856446,
-                0.02802777671813965,
-                0.028034048080444338,
-                0.028017663955688478,
-                0.02799308776855469,
-                0.02798080062866211,
-                0.027991039276123047,
-                0.027995136260986327,
-                0.027984064102172853,
-                0.027921215057373047,
-                0.02797670364379883,
-                0.02800230407714844,
-                0.02795724868774414,
-                0.02792550468444824,
-                0.027991039276123047,
-                0.02795622444152832,
-                0.02798489570617676,
-                0.028110847473144532,
-                0.027997184753417968,
-                0.028050432205200194,
-                0.028066816329956053,
-                0.028067840576171874,
-                0.02802617645263672,
-                0.027970239639282225,
-                0.02797260856628418,
-                0.028064767837524415,
-                0.02799001693725586,
-                0.027983871459960938,
-                0.027952159881591797,
-                0.02797667121887207,
-                0.02795110321044922,
-                0.027966463088989257,
-                0.02800230407714844,
-                0.027982847213745117,
-                0.028027904510498046,
-                0.027974655151367187,
-                0.02799827194213867,
-                0.027931583404541015,
-                0.028000320434570312,
-                0.02801043128967285,
-                0.027996160507202147,
-                0.02796953582763672,
-                0.027999359130859373
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.79977107116131
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 991.870976,
             "max_vram": 15755.902976,
             "max_reserved": 15271.460864,
             "max_allocated": 15114.469888
         },
         "latency": {
             "unit": "s",
+            "mean": 0.035889704044574464,
+            "stdev": 0.0002602956869559564,
             "values": [
+                0.03887427139282226,
+                0.03758796691894531,
+                0.03615948867797852,
+                0.036985855102539066,
+                0.036106239318847655,
                 0.036176895141601564,
+                0.03678617477416992,
+                0.03714355087280274,
+                0.0371486701965332,
+                0.036574207305908206,
+                0.03582361602783203,
+                0.03575603103637695,
+                0.03594956970214844,
+                0.03583596801757812,
+                0.03584000015258789,
+                0.03585331344604492,
+                0.03585433578491211,
                 0.03601203155517578,
+                0.03580825424194336,
+                0.03580518341064453,
+                0.035846145629882815,
+                0.03579289627075195,
+                0.03583180618286133,
+                0.03586355209350586,
+                0.03579391860961914,
+                0.035817470550537106,
+                0.03582156753540039,
+                0.03583795166015625,
+                0.035855358123779296,
+                0.035783679962158206,
+                0.03582880020141602,
+                0.035883007049560545,
+                0.03583078384399414,
+                0.03577958297729492,
+                0.03577139282226562,
+                0.035814399719238284,
+                0.0358922233581543,
+                0.03580723190307617,
+                0.035806209564208984,
+                0.03580108642578125,
+                0.03581977462768555,
+                0.03576627349853516,
+                0.035794944763183595,
+                0.0357918701171875,
+                0.035810367584228515,
+                0.03580928039550781,
+                0.03613183975219727,
+                0.0358389778137207,
+                0.035789825439453124,
+                0.03582668685913086,
+                0.03589427185058594,
+                0.035778560638427735,
+                0.03580742263793945,
+                0.03581151962280273,
+                0.03580416107177734,
+                0.035778560638427735,
+                0.03585126495361328,
+                0.035781631469726564,
+                0.03579904174804688,
+                0.0358205451965332,
+                0.03576422500610352,
+                0.03581849670410156,
+                0.035811328887939455,
                 0.035929088592529294,
+                0.03575296020507813,
+                0.0358021125793457,
+                0.03580723190307617,
+                0.03580825424194336,
+                0.035776512145996094,
+                0.03581148910522461,
+                0.035834945678710935,
+                0.03585433578491211,
+                0.03582156753540039,
+                0.03583488082885742,
+                0.03583488082885742,
                 0.03590553665161133,
+                0.03582259368896484,
+                0.035800193786621096,
+                0.03580416107177734,
+                0.035855358123779296,
+                0.03581235122680664,
+                0.035829761505126956,
+                0.03582361602783203,
+                0.03580825424194336,
+                0.03581235122680664,
+                0.035768318176269534,
+                0.03575091171264649,
+                0.03580416107177734,
+                0.035800064086914066,
+                0.035783679962158206,
+                0.03582463836669922,
+                0.035789825439453124,
+                0.03581644821166992,
+                0.03580847930908203,
+                0.03578777694702148,
+                0.03582169723510742,
+                0.03574784088134766,
+                0.035743743896484374,
+                0.035781631469726564,
+                0.03581951904296875,
+                0.03575500869750976,
+                0.035814399719238284,
+                0.03576319885253906,
+                0.03577036666870117,
+                0.03579391860961914,
+                0.03579084777832031,
+                0.03595587158203125,
+                0.03578572845458984,
+                0.0359106559753418,
+                0.035883007049560545,
+                0.0358809585571289,
+                0.03587788772583008,
+                0.03585647964477539,
+                0.03581849670410156,
+                0.03585433578491211,
+                0.035838272094726564,
+                0.03586355209350586,
+                0.03584921646118164,
+                0.03587583923339844,
+                0.0359741439819336,
                 0.035929088592529294,
+                0.035850238800048825,
+                0.035800064086914066,
+                0.03584921646118164,
+                0.03585331344604492,
+                0.035899391174316404,
+                0.03585945510864258,
+                0.03587071990966797,
+                0.03588608169555664,
+                0.0359024658203125,
+                0.03584204864501953,
+                0.03584422302246094,
+                0.03585945510864258,
+                0.03586355209350586,
+                0.03585647964477539,
+                0.035862529754638675,
+                0.035829761505126956,
+                0.03586457443237305,
+                0.03583283233642578,
+                0.03589427185058594,
+                0.03584921646118164,
+                0.03585433578491211,
+                0.035862529754638675,
+                0.035885055541992186,
+                0.035813377380371096,
+                0.03585638427734375,
+                0.03583283233642578,
+                0.035827713012695314,
+                0.035829761505126956,
+                0.035888126373291016,
+                0.035855358123779296,
+                0.03588198471069336,
+                0.0358389778137207,
                 0.03603558349609375,
+                0.035794944763183595,
+                0.03583615875244141,
+                0.03582361602783203,
+                0.03583692932128906,
+                0.03581849670410156,
+                0.035904510498046875,
                 0.03595775985717774,
+                0.03584819030761719,
                 0.03595161437988281,
+                0.035862529754638675,
+                0.035850238800048825,
+                0.03584000015258789,
+                0.03584102249145508,
+                0.0358392333984375,
+                0.03585843276977539,
+                0.035860481262207033,
+                0.035879936218261715,
+                0.03584716796875,
+                0.035806209564208984,
+                0.03584204864501953,
                 0.03590963363647461,
+                0.03610419082641601,
+                0.03589324951171875,
+                0.035833854675292966,
+                0.03585331344604492,
+                0.03585638427734375,
+                0.03578777694702148,
+                0.03582668685913086,
                 0.03586969757080078,
+                0.035792064666748044,
+                0.0359628791809082,
+                0.03585740661621094,
+                0.03585945510864258,
+                0.035851585388183595,
+                0.03584511947631836,
+                0.035827713012695314,
+                0.03590252685546875,
+                0.035862529754638675,
+                0.03583078384399414,
+                0.03586867141723633,
+                0.03580928039550781,
                 0.035883007049560545,
+                0.035862529754638675,
+                0.03586355209350586,
+                0.03588915252685547,
+                0.035904510498046875,
+                0.035871742248535156,
+                0.03588710403442383,
                 0.035942401885986325,
+                0.03615027236938476,
+                0.0358389778137207,
+                0.03586764907836914,
+                0.035885055541992186,
+                0.03583692932128906,
+                0.035846145629882815,
+                0.0358809585571289,
                 0.035947521209716796,
+                0.03588608169555664,
+                0.03592192077636719,
+                0.035899391174316404,
+                0.03586457443237305,
+                0.035862529754638675,
+                0.03585945510864258,
+                0.03589836883544922,
                 0.036055038452148434,
+                0.035888126373291016,
+                0.035932159423828124,
+                0.035844097137451174,
+                0.03585843276977539,
+                0.03584307098388672,
+                0.03586457443237305,
+                0.03586355209350586,
+                0.035810302734375,
+                0.0358737907409668,
+                0.03587481689453125,
+                0.0358737907409668,
+                0.03587891387939453,
+                0.035917823791503906,
+                0.0360263671875,
+                0.0358389778137207,
+                0.03586764907836914,
+                0.03585433578491211,
+                0.03586358261108399,
+                0.03586457443237305,
+                0.03585228729248047,
+                0.03587276840209961,
+                0.035899391174316404,
+                0.03584524917602539,
+                0.03586150360107422,
+                0.035833854675292966,
+                0.035833023071289063,
+                0.035850238800048825,
+                0.03594035339355469,
+                0.035866622924804685,
+                0.03590553665161133,
+                0.03584022521972656,
+                0.03584102249145508,
+                0.03589529418945313,
+                0.03589427185058594,
+                0.03588198471069336,
+                0.03585638427734375,
+                0.035866622924804685,
+                0.03582156753540039,
+                0.03587686538696289,
+                0.03582566452026367,
+                0.03580723190307617,
+                0.03584716796875,
+                0.0358389778137207,
+                0.03585638427734375,
+                0.03585126495361328,
+                0.03587891387939453,
+                0.0358205451965332,
+                0.03584012985229492,
+                0.03582166290283203,
+                0.03586355209350586,
+                0.03581760025024414,
+                0.03586457443237305,
+                0.03578265762329102,
+                0.035789825439453124,
+                0.035829761505126956,
+                0.03582463836669922,
+                0.035917823791503906,
+                0.03585945510864258,
+                0.03583590316772461,
+                0.03586457443237305
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 7132.964921695981
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 991.870976,
+            "max_vram": 15925.772288,
+            "max_reserved": 15441.330176,
             "max_allocated": 15238.453248
         },
         "latency": {
             "unit": "s",
+            "mean": 7.1135324239730835,
+            "stdev": 0.007382006645201944,
             "values": [
+                7.120914430618286,
+                7.106150417327882
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 35.84716914210341
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.02789620558420818,
+            "stdev": 0.0003394908393536961,
             "values": [
+                0.02979635238647461,
+                0.029270015716552734,
+                0.02769715118408203,
+                0.027704320907592773,
+                0.027724800109863282,
+                0.027677696228027345,
+                0.027681791305541992,
+                0.027644927978515626,
+                0.02772889518737793,
+                0.02776268768310547,
                 0.02794495964050293,
+                0.027890687942504884,
+                0.02774015998840332,
+                0.027695104598999022,
+                0.027691007614135742,
+                0.0276889591217041,
+                0.027677696228027345,
+                0.02771353530883789,
+                0.027720703125,
+                0.02775859260559082,
+                0.027711488723754882,
+                0.02779136085510254,
+                0.027715583801269532,
+                0.02771251106262207,
+                0.027652095794677735,
+                0.027695104598999022,
+                0.027633663177490234,
+                0.027696128845214843,
+                0.027622400283813478,
+                0.027704320907592773,
+                0.02768076705932617,
+                0.027684864044189454,
+                0.027669504165649415,
+                0.02768998336791992,
+                0.0276889591217041,
+                0.02776678466796875,
+                0.02769715118408203,
+                0.02771353530883789,
+                0.027682815551757813,
                 0.027926528930664062,
+                0.02770227241516113,
+                0.027807743072509765,
+                0.027847679138183593,
+                0.02776473617553711,
+                0.02792857551574707,
+                0.027778047561645508,
+                0.02770227241516113,
+                0.027677696228027345,
+                0.02773811149597168,
+                0.027684864044189454,
+                0.02774732780456543,
+                0.027775999069213866,
+                0.027693056106567384,
+                0.027707391738891602,
+                0.027670528411865233,
+                0.02768998336791992,
+                0.02773504066467285,
                 0.027868160247802733,
+                0.027922431945800782,
+                0.02775449562072754,
+                0.027704320907592773,
+                0.02772889518737793,
+                0.02774323272705078,
+                0.027707391738891602,
+                0.02771968078613281,
+                0.027792383193969726,
+                0.027926528930664062,
+                0.02773606491088867,
+                0.027782144546508788,
+                0.02774220848083496,
+                0.02775654411315918,
+                0.02774323272705078,
+                0.027685888290405275,
+                0.02774015998840332,
+                0.027732992172241212,
+                0.02780467224121094,
+                0.02772684860229492,
+                0.027792383193969726,
+                0.027792383193969726,
+                0.027711488723754882,
+                0.02771865653991699,
+                0.02777497673034668,
+                0.027812864303588865,
                 0.02792755126953125,
+                0.027691007614135742,
+                0.02844700813293457,
+                0.028763872146606445,
+                0.027752447128295898,
+                0.02772787284851074,
+                0.027780223846435546,
+                0.027777919769287108,
                 0.027909120559692382,
+                0.027828224182128908,
+                0.027778047561645508,
+                0.028133567810058595,
+                0.027829055786132813,
+                0.02778726387023926,
+                0.027789312362670897,
+                0.027811840057373048,
+                0.02774630355834961,
+                0.02777907180786133,
+                0.02775142478942871,
+                0.02774835205078125,
+                0.027814912796020507,
+                0.027834367752075196,
+                0.027832319259643554,
+                0.027810815811157227,
                 0.027859968185424806,
+                0.027789312362670897,
+                0.027808767318725586,
+                0.027807743072509765,
+                0.02777190399169922,
+                0.027825151443481445,
+                0.027834367752075196,
+                0.02779136085510254,
+                0.02780364799499512,
+                0.027843584060668947,
+                0.027785215377807617,
+                0.027800575256347656,
+                0.02778316879272461,
+                0.027832319259643554,
+                0.028072959899902345,
+                0.029672447204589843,
+                0.029611007690429687,
+                0.029549568176269532,
+                0.02956185531616211,
+                0.029363199234008788,
+                0.029443071365356444,
+                0.029454336166381836,
+                0.029418495178222655,
+                0.029244415283203123,
+                0.02935091209411621,
+                0.029351936340332032,
+                0.02877542304992676,
+                0.028625919342041017,
+                0.0285665283203125,
+                0.028281856536865234,
+                0.02898841667175293,
+                0.028799999237060548,
+                0.028820480346679687,
+                0.02838528060913086,
+                0.028313600540161132,
+                0.027885568618774413,
+                0.027790336608886718,
+                0.02773504066467285,
+                0.02773708724975586,
+                0.02773504066467285,
+                0.02776473617553711,
+                0.027813888549804686,
+                0.027807743072509765,
+                0.027782144546508788,
+                0.02775040054321289,
+                0.02778828811645508,
+                0.02774323272705078,
+                0.027871231079101562,
                 0.027910144805908203,
+                0.027786239624023438,
+                0.02772684860229492,
+                0.02779955291748047,
+                0.02777497673034668,
+                0.027800575256347656,
+                0.027838464736938476,
+                0.027812864303588865,
+                0.027790336608886718,
+                0.027832319259643554,
+                0.027793407440185547,
+                0.027757568359375,
                 0.027857919692993165,
+                0.027828224182128908,
+                0.027841535568237305,
+                0.027842559814453126,
                 0.027869184494018553,
+                0.027786272048950195,
+                0.02779644775390625,
+                0.02775859260559082,
+                0.027802623748779298,
+                0.02775347137451172,
+                0.02778726387023926,
+                0.027810815811157227,
+                0.027851776123046876,
+                0.02778828811645508,
+                0.027815935134887695,
                 0.02791628837585449,
+                0.02777497673034668,
+                0.027870208740234374,
+                0.027901952743530273,
+                0.027831296920776367,
+                0.02786832046508789,
+                0.027791200637817384,
                 0.02780467224121094,
+                0.027892736434936522,
+                0.02795622444152832,
+                0.027889663696289063,
+                0.02794905662536621,
+                0.027890687942504884,
+                0.027765760421752928,
+                0.027834367752075196,
+                0.027807743072509765,
+                0.027809791564941407,
+                0.027842559814453126,
+                0.027858943939208985,
+                0.027844608306884764,
+                0.027768831253051757,
+                0.02777497673034668,
+                0.028292095184326172,
+                0.027883520126342775,
+                0.027823104858398437,
+                0.027825151443481445,
+                0.027867136001586915,
                 0.027893760681152343,
+                0.028031999588012696,
+                0.02792140769958496,
+                0.027871231079101562,
+                0.027865087509155274,
+                0.027844608306884764,
+                0.027870208740234374,
+                0.027877376556396483,
+                0.027894784927368164,
+                0.027859039306640625,
+                0.027866016387939452,
                 0.02788761520385742,
+                0.027884544372558592,
+                0.027829248428344725,
+                0.027881471633911133,
                 0.027892736434936522,
+                0.027858943939208985,
+                0.02791116714477539,
+                0.027968767166137696,
+                0.027831039428710937,
                 0.027833343505859375,
+                0.027836416244506838,
+                0.027880447387695313,
                 0.027897855758666993,
+                0.027824127197265625,
+                0.027870208740234374,
+                0.02789580726623535,
+                0.027873279571533204,
+                0.027884544372558592,
                 0.02790399932861328,
+                0.027917375564575197,
+                0.02787833595275879,
+                0.02814668846130371,
+                0.02796441650390625,
                 0.027886592864990234,
+                0.027869184494018553,
+                0.02791731262207031,
+                0.027923456192016603,
+                0.027869184494018553,
+                0.02778112030029297,
                 0.027888639450073242,
+                0.02788761520385742,
+                0.02776268768310547,
+                0.02776166343688965,
+                0.027845632553100585,
+                0.027859968185424806,
+                0.02774732780456543,
+                0.027701248168945314,
+                0.02771455955505371,
+                0.027858943939208985,
+                0.02775040054321289,
+                0.027687936782836913,
+                0.030300159454345704,
+                0.02932633590698242,
+                0.029247488021850586,
+                0.029257728576660157,
+                0.028466175079345703,
+                0.027864063262939453,
+                0.02772787284851074,
+                0.027720703125,
+                0.027716608047485353,
+                0.02770534324645996,
+                0.02773606491088867,
+                0.02771353530883789,
+                0.027716608047485353,
+                0.02774220848083496,
+                0.027682016372680664,
+                0.027821855545043947,
+                0.027792383193969726,
+                0.02772377586364746,
+                0.02770636749267578,
+                0.027881471633911133,
+                0.027813888549804686,
+                0.02770249557495117,
+                0.027681568145751952,
+                0.027686912536621092,
+                0.0276889591217041,
+                0.027655168533325194,
+                0.02771353530883789,
+                0.02772889518737793,
+                0.027648000717163085,
+                0.027679744720458983,
+                0.02772172737121582,
+                0.027677696228027345,
+                0.02777497673034668,
+                0.028104703903198244,
+                0.02775654411315918,
+                0.027699199676513672,
+                0.027717632293701173,
+                0.028262399673461915,
+                0.028090368270874022,
+                0.02774220848083496,
+                0.02771251106262207,
+                0.027686912536621092,
+                0.027844608306884764,
+                0.028417024612426758,
+                0.02773401641845703,
+                0.027809791564941407,
+                0.027822080612182616,
+                0.027631616592407225,
+                0.02779136085510254,
+                0.02776371192932129,
+                0.027816959381103516,
+                0.02773196792602539,
+                0.027834367752075196,
+                0.02776780891418457,
+                0.02773811149597168,
+                0.02773401641845703,
+                0.02770636749267578,
+                0.02775040054321289,
+                0.02772991943359375,
+                0.02772991943359375,
+                0.02775551986694336,
+                0.02796953582763672,
+                0.02776678466796875,
+                0.027837440490722655,
+                0.0277708797454834,
+                0.027823104858398437,
+                0.02774732780456543,
+                0.02776678466796875,
+                0.02776473617553711,
+                0.027809791564941407,
+                0.027829248428344725,
+                0.02778726387023926,
+                0.027859968185424806,
+                0.02777292823791504,
+                0.027752447128295898,
+                0.027848703384399414,
+                0.027812864303588865,
+                0.027812864303588865,
+                0.027695104598999022,
+                0.02779136085510254,
+                0.027807743072509765,
+                0.02775449562072754,
+                0.027782144546508788,
+                0.027790336608886718,
+                0.02775449562072754,
+                0.02776473617553711,
+                0.02776166343688965,
+                0.027773952484130858,
+                0.027811840057373048,
+                0.02773504066467285,
+                0.027827199935913087,
+                0.027768831253051757,
+                0.028128255844116212,
+                0.02821347236633301,
+                0.027877151489257814,
+                0.027838464736938476,
+                0.02775449562072754,
+                0.02774323272705078,
+                0.02776371192932129,
+                0.02773504066467285,
+                0.027806720733642577,
+                0.027807743072509765,
+                0.02793984031677246,
+                0.02779955291748047,
+                0.027794431686401368,
+                0.02774732780456543,
+                0.027833343505859375,
+                0.02780467224121094,
+                0.027825151443481445,
+                0.02800230407714844,
+                0.02776678466796875,
+                0.02773606491088867,
+                0.02778009605407715,
                 0.027883520126342775,
+                0.027817983627319336,
+                0.027855871200561523,
+                0.027778047561645508,
+                0.027809791564941407,
+                0.0277391357421875,
+                0.027813888549804686,
+                0.02776678466796875,
+                0.02779545593261719,
+                0.027801599502563477,
+                0.027827199935913087,
+                0.02770636749267578,
+                0.027828224182128908,
+                0.02774835205078125,
+                0.027782144546508788,
+                0.027801599502563477,
+                0.027811840057373048,
+                0.02778112030029297,
+                0.02773708724975586,
+                0.027828224182128908,
+                0.02778726387023926,
+                0.027860992431640624,
+                0.02779136085510254,
+                0.02778316879272461,
+                0.02774835205078125,
+                0.027827199935913087,
+                0.027802623748779298,
+                0.02779136085510254,
+                0.02780467224121094,
+                0.02775961685180664,
+                0.027890687942504884,
+                0.02810572814941406,
                 0.027851776123046876,
+                0.02775040054321289,
+                0.027831296920776367,
+                0.027846656799316406,
+                0.027837440490722655,
+                0.027846656799316406,
+                0.02778726387023926,
+                0.027947008132934572,
+                0.027822080612182616,
+                0.027816959381103516,
+                0.027797504425048827,
+                0.027808767318725586,
+                0.02775142478942871,
+                0.02777190399169922,
+                0.02778828811645508,
+                0.027862016677856444,
+                0.027822240829467774,
+                0.02781782341003418,
                 0.027839487075805663,
+                0.02797670364379883,
+                0.027860992431640624,
+                0.027831296920776367,
+                0.02794086456298828,
+                0.027807743072509765,
+                0.027870208740234374,
+                0.027808767318725586,
+                0.028061695098876953,
+                0.027848703384399414,
+                0.02779136085510254,
+                0.027872255325317383,
+                0.02789990425109863,
+                0.027866111755371094,
+                0.027846656799316406,
+                0.027815935134887695,
+                0.027854848861694335,
+                0.027807743072509765,
+                0.027833343505859375,
+                0.02815590476989746,
+                0.027888639450073242,
+                0.027859968185424806,
+                0.027854848861694335,
+                0.027829248428344725,
+                0.027869184494018553,
+                0.027846656799316406,
+                0.02792140769958496,
+                0.028218368530273437,
+                0.027825151443481445,
+                0.027825151443481445,
                 0.027851776123046876,
+                0.027836416244506838,
+                0.027831296920776367,
+                0.027844608306884764,
+                0.027784191131591796,
+                0.027842559814453126,
+                0.027840511322021484,
+                0.027844608306884764,
+                0.02779955291748047,
+                0.027909120559692382,
+                0.027830272674560546,
+                0.027865087509155274,
+                0.027880447387695313,
+                0.027805696487426756,
+                0.027869184494018553,
+                0.027868160247802733,
+                0.027830272674560546,
                 0.027889663696289063,
+                0.02793574333190918,
+                0.02783036804199219,
+                0.027938720703125,
+                0.027871231079101562,
+                0.027856895446777344,
+                0.027852800369262694,
+                0.027863040924072265,
+                0.027862016677856444,
+                0.027913215637207032,
+                0.027872255325317383,
+                0.027871231079101562,
+                0.027827199935913087,
+                0.027820032119750978,
+                0.027820032119750978,
+                0.027813888549804686,
+                0.028193792343139647,
+                0.028027904510498046,
+                0.028009471893310548,
+                0.028003328323364256,
+                0.028076032638549804,
+                0.028169216156005858,
+                0.027986944198608397,
+                0.02797056007385254,
+                0.028003328323364256,
+                0.028045312881469726,
+                0.028087295532226563,
+                0.028251136779785156,
+                0.028099584579467773,
                 0.028044288635253906,
+                0.028075008392333983,
+                0.028051679611206054,
+                0.028056352615356446,
                 0.028071935653686524,
+                0.028041215896606447,
+                0.02797772789001465,
+                0.028070911407470703,
+                0.02794495964050293,
+                0.02795212745666504
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 35.847169142103404
         },
         "energy": null,
         "efficiency": null