kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Apr 16

Commit

9d05844

•

1 Parent(s): d7f154c

Upload /gemma-7b/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

gemma-7b/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +800 -803

gemma-7b/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,306 +2,303 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1039.458304,
-            "max_vram": 17811.111936,
             "max_reserved": 17733.517312,
             "max_allocated": 17665.339904
         },
         "latency": {
             "unit": "s",
-            "mean": 0.03527099115472092,
-            "stdev": 0.0001648795417007636,
             "values": [
-                0.03710812759399414,
-                0.03495731353759766,
-                0.035369983673095705,
-                0.03508224105834961,
-                0.03496857452392578,
-                0.03523388671875,
-                0.03534950256347656,
-                0.035297279357910154,
-                0.03517871856689453,
-                0.03505254364013672,
-                0.034966529846191405,
-                0.03497593688964844,
-                0.03491328048706055,
-                0.034953216552734374,
-                0.034939903259277344,
-                0.03496243286132812,
-                0.03492761611938477,
-                0.03497574234008789,
-                0.03492966461181641,
-                0.034945022583007815,
-                0.0349224967956543,
-                0.034948097229003904,
-                0.034920448303222655,
-                0.034991104125976565,
-                0.03508838272094727,
-                0.03561881637573242,
-                0.03564035034179688,
-                0.03561164855957031,
-                0.03562188720703125,
-                0.03561574554443359,
-                0.03561471939086914,
-                0.03563622283935547,
-                0.035622913360595705,
-                0.035555328369140625,
-                0.03552665710449219,
-                0.03533238220214844,
-                0.03526041412353516,
-                0.03530672073364258,
-                0.035141632080078124,
-                0.03514572906494141,
-                0.035094657897949216,
-                0.03510476684570313,
-                0.03510291290283203,
-                0.03507712173461914,
-                0.035064830780029296,
-                0.035064830780029296,
-                0.03502489471435547,
-                0.035043327331542966,
-                0.035037185668945314,
-                0.035059711456298825,
-                0.03503308868408203,
-                0.035211265563964846,
-                0.03519180679321289,
-                0.035198974609375,
-                0.03520204925537109,
-                0.035348480224609374,
-                0.03546316909790039,
-                0.03546931076049804,
-                0.03545600128173828,
-                0.03549184036254883,
-                0.035489791870117186,
-                0.03548672103881836,
-                0.035402751922607424,
                 0.03542732620239258,
-                0.035345409393310545,
-                0.0353331184387207,
-                0.03527065658569336,
-                0.03530854415893555,
-                0.03517542266845703,
-                0.0352042236328125,
-                0.03514470291137695,
-                0.03508633422851563,
-                0.03514572906494141,
-                0.035097599029541016,
-                0.03513958358764648,
-                0.035149822235107424,
-                0.03514371109008789,
-                0.0351539192199707,
-                0.03511808013916016,
-                0.035168254852294925,
-                0.03522969436645508,
-                0.0352542724609375,
-                0.03525939178466797,
-                0.035264511108398434,
-                0.03527167892456055,
-                0.03529216003417969,
-                0.03534950256347656,
-                0.03535871887207031,
-                0.035345409393310545,
-                0.03545600128173828,
-                0.035364864349365234,
-                0.03544473648071289,
-                0.03535171127319336,
-                0.0353361930847168,
-                0.035336254119873046,
                 0.035329025268554685,
-                0.03532799911499023,
-                0.035340286254882815,
-                0.03525763320922851,
-                0.035280895233154294,
-                0.03522969436645508,
-                0.03524300765991211,
-                0.03518585586547852,
-                0.035209217071533204,
-                0.03521023941040039,
-                0.0351539192199707,
-                0.035143680572509765,
-                0.03524915313720703,
-                0.03524198532104492,
-                0.03526553726196289,
-                0.03524915313720703,
-                0.03528192138671875,
-                0.035272705078125,
-                0.03530342483520508,
-                0.035266559600830076,
-                0.035302654266357425,
-                0.03526259231567383,
-                0.03529334259033203,
-                0.03528806304931641,
-                0.03529420852661133,
-                0.03535257720947266,
-                0.035361793518066405,
-                0.03532304000854492,
-                0.035353599548339845,
-                0.03533004760742187,
-                0.035351551055908204,
-                0.0353361930847168,
                 0.03535974502563476,
-                0.03522252655029297,
-                0.03526348876953125,
-                0.035198974609375,
-                0.03523891067504883,
-                0.03526144027709961,
-                0.035249248504638675,
-                0.035244033813476565,
-                0.03528403091430664,
-                0.03522054290771484,
-                0.035253246307373046,
-                0.035269630432128905,
-                0.03530342483520508,
-                0.03528908920288086,
-                0.03528499221801758,
-                0.03527475357055664,
-                0.035266559600830076,
-                0.03527475357055664,
-                0.035296257019042966,
-                0.035296257019042966,
-                0.03530649566650391,
-                0.035282943725585936,
-                0.03531673431396484,
-                0.03528396987915039,
-                0.03550822448730469,
-                0.035278079986572265,
-                0.03528192138671875,
-                0.03528192138671875,
-                0.03529011154174805,
-                0.035227649688720705,
-                0.03531468963623047,
-                0.03523993682861328,
-                0.03524198532104492,
-                0.03524095916748047,
-                0.035266559600830076,
-                0.03527577590942383,
-                0.035250175476074216,
-                0.035261695861816406,
-                0.03527577590942383,
-                0.035286014556884765,
-                0.03527475357055664,
-                0.035264511108398434,
-                0.03530659103393555,
-                0.03527372741699219,
-                0.03531980895996094,
-                0.03527372741699219,
-                0.03530137634277344,
-                0.035282943725585936,
-                0.0352911376953125,
-                0.03527679824829102,
-                0.03527167892456055,
-                0.03527782440185547,
-                0.03529523086547852,
-                0.03526348876953125,
-                0.03530342483520508,
-                0.03527372741699219,
-                0.035279872894287106,
-                0.03528806304931641,
-                0.03528396987915039,
-                0.03530137634277344,
-                0.035244033813476565,
-                0.0352553596496582,
-                0.03526553726196289,
-                0.03524198532104492,
-                0.035302398681640625,
-                0.0352911376953125,
-                0.035286014556884765,
-                0.03525939178466797,
-                0.035315711975097655,
-                0.035280895233154294,
-                0.03529318237304688,
-                0.035269630432128905,
-                0.035302398681640625,
-                0.03529216003417969,
-                0.035280895233154294,
-                0.03528704071044922,
-                0.035383296966552735,
-                0.03527679824829102,
-                0.03532287979125977,
-                0.03526041412353516,
-                0.035268798828125,
-                0.03528499221801758,
-                0.035297279357910154,
-                0.03527475357055664,
-                0.035266559600830076,
-                0.03528806304931641,
-                0.035315711975097655,
-                0.035254337310791015,
-                0.03526150512695313,
-                0.03530547332763672,
-                0.035264511108398434,
-                0.03528499221801758,
-                0.03524095916748047,
-                0.03529830551147461,
-                0.035286014556884765,
-                0.03529523086547852,
-                0.035302398681640625,
-                0.035286014556884765,
-                0.035297279357910154,
-                0.03528908920288086,
-                0.035282943725585936,
-                0.0352542724609375,
-                0.03529011154174805,
-                0.03528396987915039,
-                0.03529523086547852,
-                0.03527884674072266,
-                0.035320831298828126,
-                0.03527167892456055,
-                0.0353177604675293,
-                0.03524732971191406,
-                0.03530547332763672,
-                0.035282943725585936,
-                0.03529216003417969,
-                0.03525734329223633,
-                0.03526041412353516,
-                0.03528499221801758,
-                0.035299327850341795,
-                0.03525120162963867,
-                0.03524095916748047,
-                0.035288192749023437,
-                0.03529011154174805,
-                0.03525939178466797,
                 0.035337215423583986,
-                0.035279872894287106,
-                0.03532198333740234,
-                0.03529420852661133,
-                0.0352542724609375,
-                0.03528908920288086,
-                0.03530342483520508,
-                0.03526758575439453,
-                0.03526758575439453,
-                0.035253246307373046,
-                0.03530956649780274,
-                0.03529318237304688,
-                0.03528396987915039,
-                0.03525632095336914,
-                0.035296257019042966,
-                0.03527475357055664,
-                0.03530137634277344,
-                0.03527577590942383,
-                0.035313697814941404,
-                0.03528499221801758,
-                0.035296257019042966,
-                0.03528192138671875,
-                0.03530271911621094,
-                0.03529523086547852,
-                0.035304447174072266,
-                0.035282943725585936,
-                0.035279872894287106,
-                0.0352911376953125,
-                0.0353177604675293,
-                0.03528499221801758,
-                0.035282943725585936,
-                0.03527679824829102,
-                0.03529830551147461,
-                0.03527577590942383,
-                0.03527372741699219,
-                0.03528499221801758
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 7258.089200754857
         },
         "energy": null,
         "efficiency": null
@@ -309,23 +306,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 1039.458304,
             "max_vram": 18593.349632,
             "max_reserved": 18108.90752,
             "max_allocated": 18044.931584
         },
         "latency": {
             "unit": "s",
-            "mean": 5.9707059202194195,
-            "stdev": 0.0009773912429760223,
             "values": [
-                5.971683311462396,
-                5.969728528976444
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 42.708517787898174
         },
         "energy": null,
         "efficiency": null
@@ -334,524 +331,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.023414533020468312,
-            "stdev": 9.766183668448149e-05,
             "values": [
-                0.02347929573059082,
-                0.02327756881713867,
-                0.02328985595703125,
-                0.02328883171081543,
-                0.023274496078491212,
-                0.023299072265625,
-                0.023267328262329103,
-                0.02326527976989746,
-                0.02327552032470703,
-                0.02327142333984375,
-                0.023282688140869142,
-                0.02327654457092285,
-                0.0232857608795166,
-                0.02327654457092285,
-                0.023266304016113282,
-                0.023266304016113282,
-                0.02327244758605957,
-                0.02326425552368164,
-                0.02326323127746582,
-                0.02326937675476074,
-                0.02328166389465332,
-                0.02327654457092285,
-                0.02326835250854492,
-                0.02328166389465332,
-                0.02328166389465332,
-                0.02327347183227539,
-                0.023274496078491212,
-                0.02328883171081543,
-                0.02327961540222168,
-                0.02327654457092285,
-                0.023274496078491212,
-                0.023282688140869142,
-                0.02327552032470703,
-                0.02329599952697754,
-                0.02328473663330078,
-                0.02328985595703125,
-                0.023290880203247072,
-                0.02328678321838379,
-                0.02329190444946289,
-                0.023310335159301757,
-                0.02329292869567871,
-                0.02330419158935547,
-                0.02330419158935547,
-                0.02329497528076172,
-                0.02329702377319336,
-                0.02330009651184082,
-                0.02329702377319336,
-                0.02330316734313965,
-                0.023307264328002928,
-                0.02330419158935547,
-                0.02330828857421875,
-                0.023315456390380858,
-                0.023302143096923827,
-                0.023310335159301757,
-                0.023302143096923827,
-                0.023327743530273438,
-                0.023304384231567384,
-                0.023310144424438475,
-                0.02330828857421875,
-                0.023323648452758788,
-                0.02331648063659668,
-                0.02332159996032715,
-                0.02330828857421875,
-                0.02331443214416504,
-                0.023318527221679687,
-                0.02332057571411133,
-                0.023333887100219726,
-                0.023347200393676756,
-                0.02332979202270508,
-                0.023334911346435547,
-                0.023343103408813477,
-                0.023341056823730468,
-                0.023340032577514647,
-                0.023353343963623048,
-                0.023345151901245118,
-                0.023365631103515624,
-                0.023356416702270507,
-                0.023349248886108398,
-                0.023342079162597656,
-                0.023349248886108398,
-                0.02333695983886719,
-                0.023359487533569336,
-                0.023344127655029297,
-                0.023362560272216795,
-                0.02334617614746094,
-                0.023357440948486328,
-                0.023355392456054686,
-                0.023352319717407227,
-                0.023356416702270507,
-                0.023365631103515624,
-                0.023353343963623048,
-                0.023365631103515624,
-                0.023357440948486328,
-                0.023361536026000978,
-                0.023353439331054687,
-                0.023361440658569335,
-                0.023358463287353515,
-                0.023363584518432616,
-                0.023373823165893554,
-                0.023379968643188476,
-                0.023360511779785157,
-                0.023377920150756838,
-                0.023369728088378908,
-                0.023560192108154295,
-                0.023372800827026367,
-                0.023400447845458985,
-                0.023365631103515624,
-                0.023399423599243165,
-                0.023368703842163087,
-                0.023393280029296876,
-                0.023388160705566406,
-                0.023405567169189453,
-                0.023378944396972655,
-                0.023402496337890624,
-                0.023385087966918947,
-                0.023392255783081056,
-                0.023383039474487305,
-                0.023403520584106444,
-                0.023387136459350585,
-                0.023393280029296876,
-                0.023374847412109375,
-                0.023399423599243165,
-                0.023382015228271484,
-                0.023382015228271484,
-                0.023393280029296876,
-                0.023396352767944335,
-                0.023391231536865235,
-                0.023398399353027344,
-                0.023398399353027344,
-                0.023408639907836915,
-                0.023405567169189453,
-                0.023426048278808592,
-                0.023410688400268553,
-                0.023413759231567383,
-                0.023414783477783203,
-                0.023418880462646483,
-                0.023414783477783203,
-                0.023407615661621094,
-                0.02344960021972656,
-                0.023418880462646483,
-                0.02342911911010742,
-                0.023423999786376954,
-                0.023421951293945312,
-                0.023422975540161133,
-                0.023430143356323242,
-                0.023432191848754884,
-                0.023419904708862304,
-                0.023940095901489256,
-                0.02352332878112793,
                 0.023601152420043944,
-                0.023589887619018556,
-                0.023650304794311523,
-                0.023550975799560548,
-                0.02353049659729004,
-                0.023547903060913086,
-                0.023563264846801758,
-                0.023517183303833008,
-                0.02351923179626465,
-                0.023438335418701172,
-                0.023537664413452147,
-                0.02352639961242676,
-                0.023538688659667968,
-                0.023549951553344727,
-                0.023456768035888673,
-                0.02345471954345703,
-                0.02345779228210449,
-                0.023548927307128906,
-                0.02353459167480469,
-                0.023536640167236327,
-                0.023575551986694337,
-                0.023532543182373047,
-                0.02347724723815918,
-                0.023448575973510744,
-                0.02346086311340332,
-                0.02346291160583496,
-                0.02345267105102539,
-                0.02345471954345703,
-                0.023463935852050782,
-                0.02347007942199707,
-                0.02349056053161621,
-                0.02347315216064453,
-                0.02346086311340332,
-                0.02346598434448242,
-                0.02346086311340332,
-                0.02347417640686035,
-                0.023478271484375,
-                0.02347007942199707,
-                0.023475488662719726,
-                0.023476959228515625,
-                0.02347007942199707,
-                0.02347007942199707,
-                0.023468032836914062,
-                0.02348236846923828,
-                0.02348748779296875,
-                0.02352230453491211,
-                0.02349875259399414,
-                0.02349977684020996,
-                0.02348134422302246,
-                0.023500799179077148,
-                0.02349363136291504,
-                0.0235100154876709,
-                0.02350387191772461,
-                0.0235100154876709,
-                0.02349772834777832,
-                0.02350387191772461,
-                0.02349875259399414,
-                0.023517183303833008,
-                0.02350182342529297,
-                0.02351513671875,
-                0.02350694465637207,
-                0.02351103973388672,
-                0.02350694465637207,
-                0.02352025604248047,
-                0.02350592041015625,
-                0.023508991241455078,
-                0.023504896163940428,
-                0.023524351119995117,
-                0.02351103973388672,
-                0.023538688659667968,
-                0.023517183303833008,
-                0.023532543182373047,
-                0.02349977684020996,
-                0.02353049659729004,
-                0.023504896163940428,
-                0.023521280288696288,
-                0.023525375366210938,
-                0.023550975799560548,
-                0.02352025604248047,
-                0.023536640167236327,
-                0.02352639961242676,
-                0.02354380798339844,
-                0.02352639961242676,
-                0.023546880722045898,
                 0.023642112731933593,
-                0.023550975799560548,
-                0.02353459167480469,
-                0.02354380798339844,
-                0.023532543182373047,
-                0.023555072784423828,
-                0.02353459167480469,
-                0.023575551986694337,
-                0.023547903060913086,
-                0.023546880722045898,
-                0.02352639961242676,
-                0.023568384170532225,
-                0.023549951553344727,
-                0.023547903060913086,
-                0.023558399200439454,
-                0.023572223663330078,
-                0.023544832229614256,
-                0.023557119369506836,
-                0.023538688659667968,
-                0.023569408416748046,
-                0.023558143615722657,
-                0.023567359924316408,
-                0.023398399353027344,
-                0.02325503921508789,
-                0.02326937675476074,
-                0.02326016044616699,
-                0.02326937675476074,
-                0.023250944137573244,
-                0.023254016876220703,
-                0.02326835250854492,
-                0.02325503921508789,
-                0.02327039909362793,
-                0.02327244758605957,
-                0.02327756881713867,
-                0.02327142333984375,
-                0.023267328262329103,
-                0.02326527976989746,
-                0.02326835250854492,
-                0.02325299263000488,
-                0.02326118469238281,
-                0.02326425552368164,
-                0.02327859115600586,
-                0.02327654457092285,
-                0.02327961540222168,
-                0.023282688140869142,
-                0.02327039909362793,
-                0.02327552032470703,
-                0.02328780746459961,
-                0.023290880203247072,
-                0.023290880203247072,
-                0.02326118469238281,
-                0.02329804801940918,
-                0.02328780746459961,
-                0.02327347183227539,
-                0.02328473663330078,
-                0.02328678321838379,
-                0.0232857608795166,
-                0.02328780746459961,
-                0.02329292869567871,
-                0.02330521583557129,
-                0.02330316734313965,
-                0.023307264328002928,
-                0.02329599952697754,
-                0.023302143096923827,
-                0.023310335159301757,
-                0.02330847930908203,
-                0.023306047439575196,
-                0.02330931282043457,
-                0.023311359405517578,
-                0.023307264328002928,
-                0.023318527221679687,
-                0.023318527221679687,
-                0.02330624008178711,
-                0.023315456390380858,
-                0.02329804801940918,
-                0.0233123836517334,
-                0.02331648063659668,
-                0.02331340789794922,
-                0.023305440902709963,
-                0.023300895690917967,
-                0.023307264328002928,
-                0.0233123836517334,
-                0.02332569694519043,
-                0.02332159996032715,
-                0.02330521583557129,
-                0.023323648452758788,
-                0.023326719284057617,
-                0.023330816268920897,
-                0.023340032577514647,
-                0.023331840515136718,
-                0.023333887100219726,
-                0.023357440948486328,
-                0.02335436820983887,
-                0.023342079162597656,
-                0.023352319717407227,
-                0.023350271224975586,
-                0.023349248886108398,
-                0.023331840515136718,
-                0.023448575973510744,
-                0.02347520065307617,
-                0.02345062446594238,
-                0.023459840774536132,
-                0.02345471954345703,
-                0.023347200393676756,
-                0.023361536026000978,
-                0.023340032577514647,
-                0.023371776580810546,
-                0.023349248886108398,
-                0.023370752334594725,
-                0.023360511779785157,
-                0.02336067199707031,
-                0.023349088668823244,
-                0.023358463287353515,
-                0.023351295471191406,
-                0.023361536026000978,
-                0.023360511779785157,
-                0.023365631103515624,
-                0.023355392456054686,
-                0.023362560272216795,
-                0.023352319717407227,
-                0.023377920150756838,
-                0.023371776580810546,
-                0.023367679595947266,
-                0.023365631103515624,
-                0.023374847412109375,
-                0.023374847412109375,
-                0.023386112213134767,
-                0.023395328521728515,
-                0.023386112213134767,
-                0.023372800827026367,
-                0.023439359664916993,
-                0.023389184951782226,
-                0.023372800827026367,
-                0.023402496337890624,
-                0.023379968643188476,
-                0.023380992889404296,
-                0.023396352767944335,
-                0.023393280029296876,
-                0.023388160705566406,
-                0.023394304275512694,
-                0.023391231536865235,
-                0.023396352767944335,
-                0.023404544830322265,
-                0.023390207290649414,
-                0.023375871658325196,
-                0.023394304275512694,
-                0.023404544830322265,
-                0.023388160705566406,
-                0.023395328521728515,
-                0.023393280029296876,
-                0.023390207290649414,
-                0.023406591415405274,
-                0.02352230453491211,
-                0.023427072525024413,
-                0.023434240341186522,
-                0.023419904708862304,
-                0.023422975540161133,
-                0.023418880462646483,
-                0.023426048278808592,
-                0.023415807723999024,
-                0.023434240341186522,
-                0.023435264587402343,
-                0.023432191848754884,
-                0.02344550323486328,
-                0.023441408157348635,
-                0.023428096771240234,
-                0.023438560485839845,
-                0.02344528007507324,
-                0.023446527481079102,
-                0.023431167602539063,
-                0.023436288833618164,
-                0.023432191848754884,
-                0.023436288833618164,
-                0.023455743789672853,
-                0.023436288833618164,
-                0.023616512298583983,
-                0.02345267105102539,
-                0.023439359664916993,
-                0.02349363136291504,
-                0.023446720123291017,
-                0.023451456069946287,
-                0.023436288833618164,
-                0.023435264587402343,
-                0.02343731117248535,
-                0.023438335418701172,
-                0.023451648712158202,
-                0.02345779228210449,
-                0.023456768035888673,
-                0.02346700859069824,
-                0.023463935852050782,
-                0.02345779228210449,
-                0.02347417640686035,
-                0.02346291160583496,
-                0.02347929573059082,
-                0.02345881652832031,
-                0.02348543930053711,
-                0.0234833927154541,
-                0.02347724723815918,
-                0.02346598434448242,
-                0.023468032836914062,
-                0.023476224899291992,
-                0.02348748779296875,
-                0.02346700859069824,
-                0.02348236846923828,
-                0.02346086311340332,
-                0.02347417640686035,
-                0.02347724723815918,
-                0.02348134422302246,
-                0.023463935852050782,
-                0.0234833927154541,
-                0.02347929573059082,
-                0.0234833927154541,
-                0.02349158477783203,
-                0.02348134422302246,
-                0.0234833927154541,
-                0.0234833927154541,
-                0.02350694465637207,
-                0.02347724723815918,
-                0.02349158477783203,
-                0.02349465560913086,
-                0.02351923179626465,
-                0.023504896163940428,
-                0.02350284767150879,
-                0.02351103973388672,
-                0.02350182342529297,
-                0.02351513671875,
-                0.023537664413452147,
-                0.023488512039184572,
-                0.02349977684020996,
-                0.02350387191772461,
-                0.02350182342529297,
-                0.023508991241455078,
-                0.02350592041015625,
-                0.02350182342529297,
-                0.023517183303833008,
-                0.02351820755004883,
-                0.0235100154876709,
-                0.02349977684020996,
-                0.023504896163940428,
-                0.023504896163940428,
-                0.0235100154876709,
-                0.023504896163940428,
-                0.0235100154876709,
-                0.02351206398010254,
-                0.023538688659667968,
-                0.02351411247253418,
-                0.023516159057617187,
-                0.02351411247253418,
-                0.02353459167480469,
-                0.02351103973388672,
-                0.02351820755004883,
-                0.023524351119995117,
-                0.02352639961242676,
-                0.023538688659667968,
-                0.02352742385864258,
-                0.023525375366210938,
-                0.02352332878112793,
-                0.02353459167480469,
-                0.023536640167236327,
-                0.0235284481048584,
-                0.023546880722045898,
-                0.023561216354370116,
-                0.023536640167236327,
-                0.023546880722045898,
-                0.023549951553344727,
-                0.02353049659729004,
-                0.023542783737182618,
-                0.023538688659667968,
-                0.023544832229614256,
-                0.023536640167236327,
-                0.023557119369506836,
-                0.023544832229614256,
-                0.023544832229614256,
-                0.02353561592102051,
-                0.023545856475830077,
-                0.02353459167480469,
-                0.023556095123291015
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 42.708517787898174
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1033.199616,
+            "max_vram": 18217.959424,
             "max_reserved": 17733.517312,
             "max_allocated": 17665.339904
         },
         "latency": {
             "unit": "s",
+            "mean": 0.035620961912980316,
+            "stdev": 0.00017304639503963636,
             "values": [
+                0.0379549446105957,
+                0.03563315200805664,
+                0.0355788803100586,
+                0.035353599548339845,
+                0.03543142318725586,
+                0.035710975646972655,
+                0.0356126708984375,
+                0.03571916961669922,
+                0.03570483016967774,
+                0.03552870559692383,
                 0.03542732620239258,
+                0.0356712646484375,
+                0.03564031982421875,
+                0.035676158905029294,
+                0.0355860481262207,
+                0.0357212142944336,
+                0.03543756866455078,
+                0.03534438323974609,
+                0.035312641143798826,
+                0.035304447174072266,
+                0.035320831298828126,
                 0.035329025268554685,
+                0.035326976776123044,
+                0.035342334747314456,
+                0.03563008117675781,
+                0.035885055541992186,
+                0.03585126495361328,
+                0.035963905334472655,
+                0.03600896072387695,
+                0.03600691223144531,
+                0.03586880111694336,
+                0.03577446365356445,
+                0.03571814346313477,
+                0.0356495361328125,
+                0.035571712493896485,
+                0.035527679443359376,
+                0.03555123138427734,
+                0.035538944244384765,
+                0.035502239227294924,
+                0.03545600128173828,
+                0.03546623992919922,
+                0.03552665710449219,
+                0.035533824920654294,
+                0.035727359771728515,
+                0.03537715148925781,
+                0.03539353561401367,
+                0.035342334747314456,
+                0.03534131240844727,
                 0.03535974502563476,
                 0.035337215423583986,
+                0.03538134384155273,
+                0.035525630950927735,
+                0.035678207397460936,
+                0.03569356918334961,
+                0.03564031982421875,
+                0.035727359771728515,
+                0.03571225738525391,
+                0.03566899108886719,
+                0.03567001724243164,
+                0.03575500869750976,
+                0.03570687866210937,
+                0.035796993255615236,
+                0.035688449859619144,
+                0.035678207397460936,
+                0.035547134399414065,
+                0.0355483512878418,
+                0.03559423828125,
+                0.03556556701660156,
+                0.035542015075683595,
+                0.03552880096435547,
+                0.03552972793579102,
+                0.03546726226806641,
+                0.03549184036254883,
+                0.03549388885498047,
+                0.03540070343017578,
+                0.03551027297973633,
+                0.035386367797851564,
+                0.0353966064453125,
+                0.035489791870117186,
+                0.035579902648925785,
+                0.03550822448730469,
+                0.03564134216308594,
+                0.0355788803100586,
+                0.03565670394897461,
+                0.03564031982421875,
+                0.0356732177734375,
+                0.03566080093383789,
+                0.035688449859619144,
+                0.03564646530151367,
+                0.03570995330810547,
+                0.03562393569946289,
+                0.035675136566162106,
+                0.03566694259643555,
+                0.03559526443481445,
+                0.03556966400146484,
+                0.03558707046508789,
+                0.03561574554443359,
+                0.03560857772827149,
+                0.035555328369140625,
+                0.03556556701660156,
+                0.0355788803100586,
+                0.035490814208984374,
+                0.0355860481262207,
+                0.03547750473022461,
+                0.0355153923034668,
+                0.035492031097412106,
+                0.03549001693725586,
+                0.0355676155090332,
+                0.03569664001464844,
+                0.03553177642822265,
+                0.03565260696411133,
+                0.03567030334472656,
+                0.0355648307800293,
+                0.035683326721191407,
+                0.03560451126098633,
+                0.035762176513671876,
+                0.03565260696411133,
+                0.035645439147949216,
+                0.035745792388916016,
+                0.03566899108886719,
+                0.035642368316650394,
+                0.035604480743408204,
+                0.03566700744628906,
+                0.03561779022216797,
+                0.03563520050048828,
+                0.03562409591674805,
+                0.035533824920654294,
+                0.035659774780273434,
+                0.03558911895751953,
+                0.03563827133178711,
+                0.03556371307373047,
+                0.03559628677368164,
+                0.035506175994873046,
+                0.035625984191894534,
+                0.035604480743408204,
+                0.03555430221557617,
+                0.03558319854736328,
+                0.03560857772827149,
+                0.035592254638671876,
+                0.03552668762207031,
+                0.03555430221557617,
+                0.03564134216308594,
+                0.03560140609741211,
+                0.0357386245727539,
+                0.03559526443481445,
+                0.03568742370605469,
+                0.03562700653076172,
+                0.03562803268432617,
+                0.035645439147949216,
+                0.03557305526733399,
+                0.035675136566162106,
+                0.03562496185302735,
+                0.03561471939086914,
+                0.03565465545654297,
+                0.03559833526611328,
+                0.035681278228759765,
+                0.03562496185302735,
+                0.03562803268432617,
+                0.035568641662597655,
+                0.0357314567565918,
+                0.03560345458984375,
+                0.035664894104003905,
+                0.03556249618530274,
+                0.0356495361328125,
+                0.03563622283935547,
+                0.03561779022216797,
+                0.03561983871459961,
+                0.03559423828125,
+                0.03563315200805664,
+                0.03564031982421875,
+                0.03561369705200195,
+                0.03564441680908203,
+                0.03555859375,
+                0.03567308807373047,
+                0.03560755157470703,
+                0.035631103515625,
+                0.03558092880249023,
+                0.03563622283935547,
+                0.035643390655517575,
+                0.03565363311767578,
+                0.035705856323242184,
+                0.03558297729492187,
+                0.035604480743408204,
+                0.035683326721191407,
+                0.035655681610107424,
+                0.03565273666381836,
+                0.03571712112426758,
+                0.03561779022216797,
+                0.03561779022216797,
+                0.03564134216308594,
+                0.0355676155090332,
+                0.0356577262878418,
+                0.03559219360351563,
+                0.03564031982421875,
+                0.03567923355102539,
+                0.035606529235839846,
+                0.03562803268432617,
+                0.03562905502319336,
+                0.03562393569946289,
+                0.03560243225097656,
+                0.03561881637573242,
+                0.03565055847167969,
+                0.03560550308227539,
+                0.035622913360595705,
+                0.03558707046508789,
+                0.03566080093383789,
+                0.03561983871459961,
+                0.035643390655517575,
+                0.03562393569946289,
+                0.035729408264160156,
+                0.03563622283935547,
+                0.03568025588989258,
+                0.03565260696411133,
+                0.03562905502319336,
+                0.03570892715454101,
+                0.03560550308227539,
+                0.035639297485351565,
+                0.035661823272705076,
+                0.0356126708984375,
+                0.03573555374145508,
+                0.03561167907714844,
+                0.03561574554443359,
+                0.035560447692871096,
+                0.03564441680908203,
+                0.03561881637573242,
+                0.03561471939086914,
+                0.03562393569946289,
+                0.035577022552490234,
+                0.03563622283935547,
+                0.03559936141967773,
+                0.03563622283935547,
+                0.0356126708984375,
+                0.03564441680908203,
+                0.03564764785766601,
+                0.03562815856933594,
+                0.035694591522216795,
+                0.03561574554443359,
+                0.03571814346313477,
+                0.035631103515625,
+                0.035671295166015624,
+                0.03566796875,
+                0.0356321907043457,
+                0.03568947219848633,
+                0.03563827133178711,
+                0.03565055847167969,
+                0.035827713012695314,
+                0.035707904815673826,
+                0.035620864868164064,
+                0.03564748764038086,
+                0.035641407012939455,
+                0.03558707046508789,
+                0.03565875244140625,
+                0.035643390655517575,
+                0.03566592025756836,
+                0.03560243225097656,
+                0.03571814346313477,
+                0.03559628677368164,
+                0.03562496185302735,
+                0.03559423828125,
+                0.03555430221557617,
+                0.035625984191894534,
+                0.03563417434692383,
+                0.03563724899291992,
+                0.03566387176513672,
+                0.03560755157470703,
+                0.03570380783081055,
+                0.035645439147949216,
+                0.03566796875,
+                0.03565363311767578,
+                0.03570175933837891,
+                0.035604480743408204,
+                0.035642368316650394,
+                0.035738784790039065,
+                0.03565670394897461,
+                0.03564851379394531,
+                0.03566387176513672,
+                0.035643390655517575,
+                0.03566592025756836,
+                0.03561676788330078,
+                0.03561164855957031,
+                0.03559936141967773,
+                0.03562393569946289
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 7186.779532382962
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1033.199616,
             "max_vram": 18593.349632,
             "max_reserved": 18108.90752,
             "max_allocated": 18044.931584
         },
         "latency": {
             "unit": "s",
+            "mean": 6.060411907196045,
+            "stdev": 0.0026623992919936335,
             "values": [
+                6.057749507904051,
+                6.063074306488038
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 42.076347928961184
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.023766321204690365,
+            "stdev": 0.00023512176003425713,
             "values": [
+                0.024010751724243166,
+                0.023640064239501952,
+                0.023609344482421874,
+                0.02365132713317871,
+                0.023629823684692384,
+                0.023622655868530275,
+                0.023622655868530275,
+                0.023632896423339843,
+                0.023640064239501952,
+                0.023629823684692384,
+                0.023609344482421874,
+                0.023617536544799804,
+                0.023610368728637695,
+                0.023633920669555664,
                 0.023601152420043944,
+                0.023612415313720703,
+                0.023610368728637695,
+                0.023625728607177734,
+                0.023611391067504883,
+                0.023621631622314454,
+                0.023600128173828124,
+                0.023608320236206053,
+                0.023603200912475586,
+                0.023611391067504883,
+                0.023606271743774415,
+                0.023619583129882812,
+                0.023611391067504883,
+                0.023629823684692384,
+                0.023614463806152345,
+                0.023622655868530275,
+                0.023600128173828124,
+                0.023625728607177734,
+                0.023622655868530275,
+                0.023638015747070314,
+                0.023642112731933593,
+                0.023633920669555664,
+                0.023617536544799804,
+                0.023646207809448243,
+                0.023661567687988282,
+                0.02366054344177246,
+                0.02365132713317871,
+                0.023653375625610353,
+                0.023634944915771484,
+                0.023654399871826173,
+                0.023630847930908205,
+                0.023649280548095702,
+                0.02364313507080078,
+                0.02364313507080078,
+                0.023631872177124022,
+                0.02364723205566406,
+                0.023644159317016602,
+                0.023649280548095702,
+                0.023634944915771484,
+                0.023653375625610353,
+                0.02365132713317871,
+                0.02367180824279785,
+                0.023682048797607422,
+                0.023665664672851562,
+                0.02365132713317871,
+                0.023665664672851562,
+                0.023636991500854493,
+                0.023677120208740233,
+                0.023692096710205078,
+                0.023670783996582033,
                 0.023642112731933593,
+                0.02365951919555664,
+                0.023661567687988282,
+                0.02366873550415039,
+                0.02367692756652832,
+                0.02367180824279785,
+                0.02367283248901367,
+                0.023682048797607422,
+                0.023686143875122072,
+                0.023669759750366212,
+                0.02371993637084961,
+                0.02370150375366211,
+                0.02367692756652832,
+                0.02369843292236328,
+                0.02368819236755371,
+                0.023670783996582033,
+                0.02369945526123047,
+                0.02368921661376953,
+                0.0236810245513916,
+                0.02368307113647461,
+                0.02368409538269043,
+                0.023710720062255858,
+                0.02369638442993164,
+                0.02370150375366211,
+                0.02370457649230957,
+                0.023685312271118163,
+                0.023707456588745117,
+                0.02369638442993164,
+                0.02369126319885254,
+                0.02369740867614746,
+                0.023710720062255858,
+                0.023686143875122072,
+                0.02370969581604004,
+                0.02368819236755371,
+                0.023706623077392578,
+                0.023710720062255858,
+                0.023727104187011717,
+                0.02372812843322754,
+                0.0237260799407959,
+                0.023723007202148438,
+                0.02373017692565918,
+                0.023702527999877928,
+                0.024013824462890625,
+                0.02374553680419922,
+                0.023759872436523437,
+                0.023735296249389647,
+                0.02372403144836426,
+                0.023766016006469725,
+                0.02371788787841797,
+                0.023724159240722655,
+                0.023740287780761718,
+                0.023743488311767577,
+                0.023759872436523437,
+                0.023735296249389647,
+                0.023742464065551756,
+                0.023727104187011717,
+                0.023740415573120118,
+                0.0237260799407959,
+                0.02372812843322754,
+                0.023740415573120118,
+                0.02373017692565918,
+                0.023734272003173826,
+                0.02374963188171387,
+                0.02373734474182129,
+                0.023734272003173826,
+                0.023751680374145507,
+                0.02376192092895508,
+                0.023800832748413086,
+                0.023750656127929686,
+                0.023767040252685546,
+                0.02376192092895508,
+                0.023770111083984375,
+                0.023781375885009767,
+                0.02376313591003418,
+                0.023750463485717774,
+                0.02376313591003418,
+                0.023758655548095704,
+                0.023769088745117187,
+                0.023768064498901367,
+                0.023747583389282227,
+                0.023769088745117187,
+                0.023771135330200196,
+                0.023759872436523437,
+                0.023800832748413086,
+                0.023778303146362305,
+                0.023796735763549806,
+                0.023801855087280274,
+                0.023784448623657226,
+                0.023775232315063476,
+                0.023826431274414063,
+                0.023773183822631837,
+                0.023784448623657226,
+                0.023775232315063476,
+                0.023779327392578126,
+                0.023776256561279296,
+                0.023769088745117187,
+                0.023804927825927736,
+                0.023785472869873047,
+                0.023794687271118165,
+                0.023798784255981444,
+                0.023800832748413086,
+                0.023790592193603514,
+                0.023794687271118165,
+                0.023811071395874024,
+                0.023814144134521483,
+                0.023806976318359374,
+                0.023806976318359374,
+                0.023794687271118165,
+                0.023799808502197265,
+                0.023819263458251954,
+                0.023797760009765623,
+                0.023799808502197265,
+                0.023802879333496094,
+                0.023802879333496094,
+                0.023815168380737304,
+                0.023797760009765623,
+                0.023818239212036133,
+                0.023799808502197265,
+                0.023805952072143553,
+                0.023847936630249023,
+                0.023797760009765623,
+                0.023813119888305666,
+                0.023805952072143553,
+                0.023817216873168946,
+                0.023839744567871093,
+                0.023810047149658203,
+                0.023831552505493164,
+                0.023819263458251954,
+                0.023814144134521483,
+                0.023817216873168946,
+                0.023822336196899413,
+                0.02387455940246582,
+                0.023830528259277343,
+                0.023829504013061522,
+                0.02384588813781738,
+                0.023819263458251954,
+                0.023875583648681642,
+                0.02386534309387207,
+                0.023851007461547852,
+                0.02387660789489746,
+                0.023863296508789062,
+                0.023863296508789062,
+                0.023875583648681642,
+                0.02385817527770996,
+                0.023846912384033202,
+                0.023852031707763673,
+                0.023883775711059572,
+                0.023871488571166992,
+                0.02388479995727539,
+                0.023879680633544922,
+                0.023855104446411132,
+                0.023900159835815428,
+                0.023860223770141603,
+                0.023867391586303712,
+                0.02388991928100586,
+                0.0238786563873291,
+                0.02387763214111328,
+                0.02387763214111328,
+                0.02387353515625,
+                0.02386227226257324,
+                0.02385817527770996,
+                0.02387763214111328,
+                0.023857152938842774,
+                0.02387455940246582,
+                0.023875583648681642,
+                0.02388582420349121,
+                0.02389094352722168,
+                0.02388582420349121,
+                0.02388787269592285,
+                0.02389299201965332,
+                0.0238919677734375,
+                0.02389708709716797,
+                0.02390732765197754,
+                0.02390425682067871,
+                0.02388275146484375,
+                0.02389504051208496,
+                0.02388991928100586,
+                0.023908351898193358,
+                0.02390425682067871,
+                0.02390937614440918,
+                0.02409267234802246,
+                0.02392985534667969,
+                0.02389811134338379,
+                0.0239052791595459,
+                0.02390323257446289,
+                0.023910400390625,
+                0.023900159835815428,
+                0.02391449546813965,
+                0.02391142463684082,
+                0.023916543960571288,
+                0.023928831100463867,
+                0.02389094352722168,
+                0.023606271743774415,
+                0.023633920669555664,
+                0.023618560791015625,
+                0.023624704360961913,
+                0.023612415313720703,
+                0.023622655868530275,
+                0.023605247497558594,
+                0.023629823684692384,
+                0.023607295989990236,
+                0.023609344482421874,
+                0.023606271743774415,
+                0.023628799438476563,
+                0.02365644836425781,
+                0.023622655868530275,
+                0.023596031188964844,
+                0.023610368728637695,
+                0.023621631622314454,
+                0.023669759750366212,
+                0.023618560791015625,
+                0.023586816787719726,
+                0.023619583129882812,
+                0.023622655868530275,
+                0.023627775192260742,
+                0.023642112731933593,
+                0.023641088485717773,
+                0.023604223251342774,
+                0.02367692756652832,
+                0.023615488052368162,
+                0.023609344482421874,
+                0.023635967254638672,
+                0.023629823684692384,
+                0.02363932800292969,
+                0.023627487182617188,
+                0.02369843292236328,
+                0.023821311950683592,
+                0.023859199523925782,
+                0.02385305595397949,
+                0.02388787269592285,
+                0.023746559143066406,
+                0.023608320236206053,
+                0.023624704360961913,
+                0.023640064239501952,
+                0.023639039993286134,
+                0.02369945526123047,
+                0.023633920669555664,
+                0.0236810245513916,
+                0.02367897605895996,
+                0.023654399871826173,
+                0.023658496856689453,
+                0.023650304794311523,
+                0.023646207809448243,
+                0.02364825630187988,
+                0.02364723205566406,
+                0.023657472610473632,
+                0.023632896423339843,
+                0.023673856735229492,
+                0.02365542411804199,
+                0.023702527999877928,
+                0.02364313507080078,
+                0.0237127685546875,
+                0.02366873550415039,
+                0.02368000030517578,
+                0.02365644836425781,
+                0.023653375625610353,
+                0.02367897605895996,
+                0.02365951919555664,
+                0.02367283248901367,
+                0.023653375625610353,
+                0.02366464042663574,
+                0.023662591934204103,
+                0.02367692756652832,
+                0.023658496856689453,
+                0.02369331169128418,
+                0.023682048797607422,
+                0.0236810245513916,
+                0.02371993637084961,
+                0.02370560073852539,
+                0.023682048797607422,
+                0.023673856735229492,
+                0.023670783996582033,
+                0.02369536018371582,
+                0.02367487907409668,
+                0.02370150375366211,
+                0.023661567687988282,
+                0.02369024085998535,
+                0.023714815139770508,
+                0.02370560073852539,
+                0.02366361618041992,
+                0.02370560073852539,
+                0.0236943359375,
+                0.0237127685546875,
+                0.023677951812744142,
+                0.023686143875122072,
+                0.02368511962890625,
+                0.02369536018371582,
+                0.02370150375366211,
+                0.02370969581604004,
+                0.02369945526123047,
+                0.02370457649230957,
+                0.024169471740722655,
+                0.02369536018371582,
+                0.02371379280090332,
+                0.02371379280090332,
+                0.023718912124633788,
+                0.023739391326904297,
+                0.02372915267944336,
+                0.023738367080688477,
+                0.02370047950744629,
+                0.023736320495605468,
+                0.023723007202148438,
+                0.02374963188171387,
+                0.02371686363220215,
+                0.023768064498901367,
+                0.023714815139770508,
+                0.023735296249389647,
+                0.02373734474182129,
+                0.023746559143066406,
+                0.02370969581604004,
+                0.02373222351074219,
+                0.02372505569458008,
+                0.023734272003173826,
+                0.02372505569458008,
+                0.02375372886657715,
+                0.02371583938598633,
+                0.02372403144836426,
+                0.02372403144836426,
+                0.023743488311767577,
+                0.02373222351074219,
+                0.023738367080688477,
+                0.023751680374145507,
+                0.023769088745117187,
+                0.023766016006469725,
+                0.023775232315063476,
+                0.02374553680419922,
+                0.023754751205444336,
+                0.023757823944091795,
+                0.023780351638793946,
+                0.023775232315063476,
+                0.023772159576416017,
+                0.023760896682739258,
+                0.023790592193603514,
+                0.023757823944091795,
+                0.023772159576416017,
+                0.023758848190307616,
+                0.023797760009765623,
+                0.023778303146362305,
+                0.023789567947387694,
+                0.023815168380737304,
+                0.023783424377441405,
+                0.023773183822631837,
+                0.023775232315063476,
+                0.023760896682739258,
+                0.023804927825927736,
+                0.023774208068847655,
+                0.023795808792114258,
+                0.023770015716552736,
+                0.023784448623657226,
+                0.023790592193603514,
+                0.023777280807495117,
+                0.023781375885009767,
+                0.023793664932250977,
+                0.023785472869873047,
+                0.023796735763549806,
+                0.023804927825927736,
+                0.023812095642089845,
+                0.023790592193603514,
+                0.023802879333496094,
+                0.023808000564575195,
+                0.023810047149658203,
+                0.023799808502197265,
+                0.023814144134521483,
+                0.023815168380737304,
+                0.023823360443115234,
+                0.023805952072143553,
+                0.023841791152954102,
+                0.023810047149658203,
+                0.023823360443115234,
+                0.023822336196899413,
+                0.023833599090576172,
+                0.023813119888305666,
+                0.023835647583007814,
+                0.023806976318359374,
+                0.02384588813781738,
+                0.023805952072143553,
+                0.023826431274414063,
+                0.023818239212036133,
+                0.023840768814086914,
+                0.023814144134521483,
+                0.023851007461547852,
+                0.023860223770141603,
+                0.023825408935546875,
+                0.023827455520629884,
+                0.023812095642089845,
+                0.023832576751708984,
+                0.023830528259277343,
+                0.023825408935546875,
+                0.023842815399169923,
+                0.023840768814086914,
+                0.023867391586303712,
+                0.023836671829223634,
+                0.028597248077392577,
+                0.023831552505493164,
+                0.023793664932250977,
+                0.02384588813781738,
+                0.02386124801635742,
+                0.023864320755004883,
+                0.02407423973083496,
+                0.023830528259277343,
+                0.02387353515625,
+                0.023821311950683592,
+                0.023851007461547852,
+                0.023838720321655273,
+                0.02388582420349121,
+                0.023843839645385743,
+                0.02385817527770996,
+                0.023848960876464844,
+                0.023855104446411132,
+                0.023860223770141603,
+                0.023875583648681642,
+                0.02385305595397949,
+                0.023875583648681642,
+                0.023857152938842774,
+                0.02388172721862793,
+                0.023855104446411132,
+                0.023863296508789062,
+                0.02388275146484375,
+                0.02385817527770996,
+                0.023857152938842774,
+                0.02388275146484375,
+                0.02387046432495117,
+                0.02386944007873535,
+                0.02387455940246582,
+                0.02389708709716797,
+                0.023875583648681642,
+                0.023916543960571288,
+                0.02389094352722168,
+                0.02388172721862793,
+                0.02390118408203125,
+                0.02389299201965332,
+                0.02390220832824707,
+                0.02389913558959961,
+                0.023888896942138672,
+                0.02389299201965332,
+                0.023868415832519533,
+                0.023900159835815428,
+                0.02388684844970703,
+                0.023900159835815428,
+                0.0238919677734375,
+                0.02388582420349121,
+                0.02391347122192383,
+                0.02388991928100586,
+                0.02390323257446289,
+                0.0239052791595459,
+                0.02389913558959961
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 42.0763479289612
         },
         "energy": null,
         "efficiency": null