kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Apr 16

Commit

ed0f055

•

1 Parent(s): a1202d1

Upload /Mistral-7B-v0.1/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Mistral-7B-v0.1/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +787 -797

Mistral-7B-v0.1/fp16_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,300 +2,290 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 991.870976,
             "max_vram": 15755.902976,
             "max_reserved": 15271.460864,
             "max_allocated": 15114.469888
         },
         "latency": {
             "unit": "s",
-            "mean": 0.035889704044574464,
-            "stdev": 0.0002602956869559564,
             "values": [
-                0.03887427139282226,
-                0.03758796691894531,
-                0.03615948867797852,
-                0.036985855102539066,
-                0.036106239318847655,
-                0.036176895141601564,
-                0.03678617477416992,
                 0.03714355087280274,
                 0.0371486701965332,
-                0.036574207305908206,
-                0.03582361602783203,
-                0.03575603103637695,
-                0.03594956970214844,
-                0.03583596801757812,
-                0.03584000015258789,
-                0.03585331344604492,
-                0.03585433578491211,
-                0.03601203155517578,
-                0.03580825424194336,
-                0.03580518341064453,
-                0.035846145629882815,
-                0.03579289627075195,
-                0.03583180618286133,
-                0.03586355209350586,
-                0.03579391860961914,
-                0.035817470550537106,
-                0.03582156753540039,
-                0.03583795166015625,
-                0.035855358123779296,
-                0.035783679962158206,
-                0.03582880020141602,
-                0.035883007049560545,
-                0.03583078384399414,
-                0.03577958297729492,
-                0.03577139282226562,
-                0.035814399719238284,
-                0.0358922233581543,
-                0.03580723190307617,
-                0.035806209564208984,
-                0.03580108642578125,
-                0.03581977462768555,
-                0.03576627349853516,
-                0.035794944763183595,
-                0.0357918701171875,
-                0.035810367584228515,
-                0.03580928039550781,
-                0.03613183975219727,
-                0.0358389778137207,
-                0.035789825439453124,
-                0.03582668685913086,
-                0.03589427185058594,
-                0.035778560638427735,
-                0.03580742263793945,
-                0.03581151962280273,
-                0.03580416107177734,
-                0.035778560638427735,
-                0.03585126495361328,
-                0.035781631469726564,
-                0.03579904174804688,
-                0.0358205451965332,
-                0.03576422500610352,
-                0.03581849670410156,
-                0.035811328887939455,
-                0.035929088592529294,
-                0.03575296020507813,
-                0.0358021125793457,
-                0.03580723190307617,
-                0.03580825424194336,
-                0.035776512145996094,
-                0.03581148910522461,
-                0.035834945678710935,
-                0.03585433578491211,
-                0.03582156753540039,
-                0.03583488082885742,
-                0.03583488082885742,
-                0.03590553665161133,
-                0.03582259368896484,
-                0.035800193786621096,
-                0.03580416107177734,
-                0.035855358123779296,
-                0.03581235122680664,
-                0.035829761505126956,
-                0.03582361602783203,
-                0.03580825424194336,
-                0.03581235122680664,
-                0.035768318176269534,
-                0.03575091171264649,
-                0.03580416107177734,
-                0.035800064086914066,
-                0.035783679962158206,
-                0.03582463836669922,
-                0.035789825439453124,
-                0.03581644821166992,
-                0.03580847930908203,
-                0.03578777694702148,
-                0.03582169723510742,
-                0.03574784088134766,
-                0.035743743896484374,
-                0.035781631469726564,
-                0.03581951904296875,
-                0.03575500869750976,
-                0.035814399719238284,
-                0.03576319885253906,
-                0.03577036666870117,
-                0.03579391860961914,
-                0.03579084777832031,
-                0.03595587158203125,
-                0.03578572845458984,
-                0.0359106559753418,
-                0.035883007049560545,
-                0.0358809585571289,
-                0.03587788772583008,
-                0.03585647964477539,
-                0.03581849670410156,
-                0.03585433578491211,
-                0.035838272094726564,
-                0.03586355209350586,
-                0.03584921646118164,
-                0.03587583923339844,
-                0.0359741439819336,
-                0.035929088592529294,
-                0.035850238800048825,
-                0.035800064086914066,
-                0.03584921646118164,
-                0.03585331344604492,
-                0.035899391174316404,
-                0.03585945510864258,
-                0.03587071990966797,
-                0.03588608169555664,
-                0.0359024658203125,
-                0.03584204864501953,
-                0.03584422302246094,
-                0.03585945510864258,
-                0.03586355209350586,
-                0.03585647964477539,
-                0.035862529754638675,
-                0.035829761505126956,
-                0.03586457443237305,
-                0.03583283233642578,
-                0.03589427185058594,
-                0.03584921646118164,
-                0.03585433578491211,
-                0.035862529754638675,
-                0.035885055541992186,
-                0.035813377380371096,
-                0.03585638427734375,
-                0.03583283233642578,
-                0.035827713012695314,
-                0.035829761505126956,
-                0.035888126373291016,
-                0.035855358123779296,
-                0.03588198471069336,
-                0.0358389778137207,
-                0.03603558349609375,
-                0.035794944763183595,
-                0.03583615875244141,
-                0.03582361602783203,
-                0.03583692932128906,
-                0.03581849670410156,
-                0.035904510498046875,
-                0.03595775985717774,
-                0.03584819030761719,
-                0.03595161437988281,
-                0.035862529754638675,
-                0.035850238800048825,
-                0.03584000015258789,
-                0.03584102249145508,
-                0.0358392333984375,
-                0.03585843276977539,
-                0.035860481262207033,
-                0.035879936218261715,
-                0.03584716796875,
-                0.035806209564208984,
-                0.03584204864501953,
-                0.03590963363647461,
-                0.03610419082641601,
-                0.03589324951171875,
-                0.035833854675292966,
-                0.03585331344604492,
-                0.03585638427734375,
-                0.03578777694702148,
-                0.03582668685913086,
-                0.03586969757080078,
-                0.035792064666748044,
-                0.0359628791809082,
-                0.03585740661621094,
-                0.03585945510864258,
-                0.035851585388183595,
-                0.03584511947631836,
-                0.035827713012695314,
-                0.03590252685546875,
-                0.035862529754638675,
-                0.03583078384399414,
-                0.03586867141723633,
-                0.03580928039550781,
-                0.035883007049560545,
-                0.035862529754638675,
-                0.03586355209350586,
-                0.03588915252685547,
-                0.035904510498046875,
-                0.035871742248535156,
-                0.03588710403442383,
-                0.035942401885986325,
-                0.03615027236938476,
-                0.0358389778137207,
-                0.03586764907836914,
-                0.035885055541992186,
-                0.03583692932128906,
-                0.035846145629882815,
-                0.0358809585571289,
-                0.035947521209716796,
-                0.03588608169555664,
-                0.03592192077636719,
-                0.035899391174316404,
-                0.03586457443237305,
-                0.035862529754638675,
-                0.03585945510864258,
-                0.03589836883544922,
-                0.036055038452148434,
-                0.035888126373291016,
-                0.035932159423828124,
-                0.035844097137451174,
-                0.03585843276977539,
-                0.03584307098388672,
-                0.03586457443237305,
-                0.03586355209350586,
-                0.035810302734375,
-                0.0358737907409668,
-                0.03587481689453125,
-                0.0358737907409668,
-                0.03587891387939453,
-                0.035917823791503906,
-                0.0360263671875,
-                0.0358389778137207,
-                0.03586764907836914,
-                0.03585433578491211,
-                0.03586358261108399,
-                0.03586457443237305,
-                0.03585228729248047,
-                0.03587276840209961,
-                0.035899391174316404,
-                0.03584524917602539,
-                0.03586150360107422,
-                0.035833854675292966,
-                0.035833023071289063,
-                0.035850238800048825,
-                0.03594035339355469,
-                0.035866622924804685,
-                0.03590553665161133,
-                0.03584022521972656,
-                0.03584102249145508,
-                0.03589529418945313,
-                0.03589427185058594,
-                0.03588198471069336,
-                0.03585638427734375,
-                0.035866622924804685,
-                0.03582156753540039,
-                0.03587686538696289,
-                0.03582566452026367,
-                0.03580723190307617,
-                0.03584716796875,
-                0.0358389778137207,
-                0.03585638427734375,
-                0.03585126495361328,
-                0.03587891387939453,
-                0.0358205451965332,
-                0.03584012985229492,
-                0.03582166290283203,
-                0.03586355209350586,
-                0.03581760025024414,
-                0.03586457443237305,
-                0.03578265762329102,
-                0.035789825439453124,
-                0.035829761505126956,
-                0.03582463836669922,
-                0.035917823791503906,
-                0.03585945510864258,
-                0.03583590316772461,
-                0.03586457443237305
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 7132.964921695981
         },
         "energy": null,
         "efficiency": null
@@ -303,23 +293,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 991.870976,
             "max_vram": 15925.772288,
             "max_reserved": 15441.330176,
             "max_allocated": 15238.453248
         },
         "latency": {
             "unit": "s",
-            "mean": 7.1135324239730835,
-            "stdev": 0.007382006645201944,
             "values": [
-                7.120914430618286,
-                7.106150417327882
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.84716914210341
         },
         "energy": null,
         "efficiency": null
@@ -328,524 +318,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.02789620558420818,
-            "stdev": 0.0003394908393536961,
             "values": [
-                0.02979635238647461,
                 0.029270015716552734,
-                0.02769715118408203,
-                0.027704320907592773,
-                0.027724800109863282,
-                0.027677696228027345,
-                0.027681791305541992,
-                0.027644927978515626,
-                0.02772889518737793,
-                0.02776268768310547,
-                0.02794495964050293,
-                0.027890687942504884,
-                0.02774015998840332,
-                0.027695104598999022,
-                0.027691007614135742,
-                0.0276889591217041,
-                0.027677696228027345,
-                0.02771353530883789,
-                0.027720703125,
-                0.02775859260559082,
-                0.027711488723754882,
-                0.02779136085510254,
-                0.027715583801269532,
-                0.02771251106262207,
-                0.027652095794677735,
-                0.027695104598999022,
-                0.027633663177490234,
-                0.027696128845214843,
-                0.027622400283813478,
-                0.027704320907592773,
-                0.02768076705932617,
-                0.027684864044189454,
-                0.027669504165649415,
-                0.02768998336791992,
-                0.0276889591217041,
-                0.02776678466796875,
-                0.02769715118408203,
-                0.02771353530883789,
-                0.027682815551757813,
-                0.027926528930664062,
-                0.02770227241516113,
-                0.027807743072509765,
-                0.027847679138183593,
-                0.02776473617553711,
-                0.02792857551574707,
-                0.027778047561645508,
-                0.02770227241516113,
-                0.027677696228027345,
-                0.02773811149597168,
-                0.027684864044189454,
-                0.02774732780456543,
-                0.027775999069213866,
-                0.027693056106567384,
-                0.027707391738891602,
-                0.027670528411865233,
-                0.02768998336791992,
-                0.02773504066467285,
-                0.027868160247802733,
-                0.027922431945800782,
-                0.02775449562072754,
-                0.027704320907592773,
-                0.02772889518737793,
-                0.02774323272705078,
-                0.027707391738891602,
-                0.02771968078613281,
-                0.027792383193969726,
-                0.027926528930664062,
-                0.02773606491088867,
-                0.027782144546508788,
-                0.02774220848083496,
-                0.02775654411315918,
-                0.02774323272705078,
-                0.027685888290405275,
-                0.02774015998840332,
-                0.027732992172241212,
-                0.02780467224121094,
-                0.02772684860229492,
-                0.027792383193969726,
-                0.027792383193969726,
-                0.027711488723754882,
-                0.02771865653991699,
-                0.02777497673034668,
-                0.027812864303588865,
-                0.02792755126953125,
-                0.027691007614135742,
-                0.02844700813293457,
-                0.028763872146606445,
-                0.027752447128295898,
-                0.02772787284851074,
-                0.027780223846435546,
-                0.027777919769287108,
-                0.027909120559692382,
-                0.027828224182128908,
-                0.027778047561645508,
-                0.028133567810058595,
-                0.027829055786132813,
-                0.02778726387023926,
-                0.027789312362670897,
-                0.027811840057373048,
-                0.02774630355834961,
-                0.02777907180786133,
-                0.02775142478942871,
-                0.02774835205078125,
-                0.027814912796020507,
-                0.027834367752075196,
-                0.027832319259643554,
-                0.027810815811157227,
-                0.027859968185424806,
-                0.027789312362670897,
-                0.027808767318725586,
-                0.027807743072509765,
-                0.02777190399169922,
-                0.027825151443481445,
-                0.027834367752075196,
-                0.02779136085510254,
-                0.02780364799499512,
-                0.027843584060668947,
-                0.027785215377807617,
-                0.027800575256347656,
-                0.02778316879272461,
-                0.027832319259643554,
-                0.028072959899902345,
-                0.029672447204589843,
-                0.029611007690429687,
-                0.029549568176269532,
-                0.02956185531616211,
-                0.029363199234008788,
-                0.029443071365356444,
-                0.029454336166381836,
-                0.029418495178222655,
-                0.029244415283203123,
-                0.02935091209411621,
-                0.029351936340332032,
-                0.02877542304992676,
-                0.028625919342041017,
-                0.0285665283203125,
-                0.028281856536865234,
-                0.02898841667175293,
-                0.028799999237060548,
-                0.028820480346679687,
-                0.02838528060913086,
-                0.028313600540161132,
-                0.027885568618774413,
-                0.027790336608886718,
-                0.02773504066467285,
-                0.02773708724975586,
-                0.02773504066467285,
-                0.02776473617553711,
-                0.027813888549804686,
-                0.027807743072509765,
-                0.027782144546508788,
-                0.02775040054321289,
-                0.02778828811645508,
-                0.02774323272705078,
-                0.027871231079101562,
-                0.027910144805908203,
-                0.027786239624023438,
-                0.02772684860229492,
-                0.02779955291748047,
-                0.02777497673034668,
-                0.027800575256347656,
-                0.027838464736938476,
-                0.027812864303588865,
-                0.027790336608886718,
-                0.027832319259643554,
-                0.027793407440185547,
-                0.027757568359375,
-                0.027857919692993165,
-                0.027828224182128908,
-                0.027841535568237305,
-                0.027842559814453126,
-                0.027869184494018553,
-                0.027786272048950195,
-                0.02779644775390625,
-                0.02775859260559082,
-                0.027802623748779298,
-                0.02775347137451172,
-                0.02778726387023926,
-                0.027810815811157227,
-                0.027851776123046876,
-                0.02778828811645508,
-                0.027815935134887695,
-                0.02791628837585449,
-                0.02777497673034668,
-                0.027870208740234374,
-                0.027901952743530273,
-                0.027831296920776367,
-                0.02786832046508789,
-                0.027791200637817384,
-                0.02780467224121094,
-                0.027892736434936522,
-                0.02795622444152832,
-                0.027889663696289063,
-                0.02794905662536621,
-                0.027890687942504884,
-                0.027765760421752928,
-                0.027834367752075196,
-                0.027807743072509765,
-                0.027809791564941407,
-                0.027842559814453126,
-                0.027858943939208985,
-                0.027844608306884764,
-                0.027768831253051757,
-                0.02777497673034668,
-                0.028292095184326172,
-                0.027883520126342775,
-                0.027823104858398437,
-                0.027825151443481445,
-                0.027867136001586915,
-                0.027893760681152343,
-                0.028031999588012696,
-                0.02792140769958496,
-                0.027871231079101562,
-                0.027865087509155274,
-                0.027844608306884764,
-                0.027870208740234374,
-                0.027877376556396483,
-                0.027894784927368164,
-                0.027859039306640625,
-                0.027866016387939452,
-                0.02788761520385742,
-                0.027884544372558592,
-                0.027829248428344725,
-                0.027881471633911133,
-                0.027892736434936522,
-                0.027858943939208985,
-                0.02791116714477539,
-                0.027968767166137696,
-                0.027831039428710937,
-                0.027833343505859375,
-                0.027836416244506838,
-                0.027880447387695313,
-                0.027897855758666993,
-                0.027824127197265625,
-                0.027870208740234374,
-                0.02789580726623535,
-                0.027873279571533204,
-                0.027884544372558592,
-                0.02790399932861328,
-                0.027917375564575197,
-                0.02787833595275879,
-                0.02814668846130371,
-                0.02796441650390625,
-                0.027886592864990234,
-                0.027869184494018553,
-                0.02791731262207031,
-                0.027923456192016603,
-                0.027869184494018553,
-                0.02778112030029297,
-                0.027888639450073242,
-                0.02788761520385742,
-                0.02776268768310547,
-                0.02776166343688965,
-                0.027845632553100585,
-                0.027859968185424806,
-                0.02774732780456543,
-                0.027701248168945314,
-                0.02771455955505371,
-                0.027858943939208985,
-                0.02775040054321289,
-                0.027687936782836913,
-                0.030300159454345704,
-                0.02932633590698242,
                 0.029247488021850586,
                 0.029257728576660157,
-                0.028466175079345703,
-                0.027864063262939453,
-                0.02772787284851074,
-                0.027720703125,
-                0.027716608047485353,
-                0.02770534324645996,
-                0.02773606491088867,
-                0.02771353530883789,
-                0.027716608047485353,
-                0.02774220848083496,
-                0.027682016372680664,
-                0.027821855545043947,
-                0.027792383193969726,
-                0.02772377586364746,
-                0.02770636749267578,
-                0.027881471633911133,
-                0.027813888549804686,
-                0.02770249557495117,
-                0.027681568145751952,
-                0.027686912536621092,
-                0.0276889591217041,
-                0.027655168533325194,
-                0.02771353530883789,
-                0.02772889518737793,
-                0.027648000717163085,
-                0.027679744720458983,
-                0.02772172737121582,
-                0.027677696228027345,
-                0.02777497673034668,
-                0.028104703903198244,
-                0.02775654411315918,
-                0.027699199676513672,
-                0.027717632293701173,
-                0.028262399673461915,
-                0.028090368270874022,
-                0.02774220848083496,
-                0.02771251106262207,
-                0.027686912536621092,
-                0.027844608306884764,
-                0.028417024612426758,
-                0.02773401641845703,
-                0.027809791564941407,
-                0.027822080612182616,
-                0.027631616592407225,
-                0.02779136085510254,
-                0.02776371192932129,
-                0.027816959381103516,
-                0.02773196792602539,
-                0.027834367752075196,
-                0.02776780891418457,
-                0.02773811149597168,
-                0.02773401641845703,
-                0.02770636749267578,
-                0.02775040054321289,
-                0.02772991943359375,
-                0.02772991943359375,
-                0.02775551986694336,
-                0.02796953582763672,
-                0.02776678466796875,
-                0.027837440490722655,
-                0.0277708797454834,
-                0.027823104858398437,
-                0.02774732780456543,
-                0.02776678466796875,
-                0.02776473617553711,
-                0.027809791564941407,
-                0.027829248428344725,
-                0.02778726387023926,
-                0.027859968185424806,
-                0.02777292823791504,
-                0.027752447128295898,
-                0.027848703384399414,
-                0.027812864303588865,
-                0.027812864303588865,
-                0.027695104598999022,
-                0.02779136085510254,
-                0.027807743072509765,
-                0.02775449562072754,
-                0.027782144546508788,
-                0.027790336608886718,
-                0.02775449562072754,
-                0.02776473617553711,
-                0.02776166343688965,
-                0.027773952484130858,
-                0.027811840057373048,
-                0.02773504066467285,
-                0.027827199935913087,
-                0.027768831253051757,
-                0.028128255844116212,
-                0.02821347236633301,
-                0.027877151489257814,
-                0.027838464736938476,
-                0.02775449562072754,
-                0.02774323272705078,
-                0.02776371192932129,
-                0.02773504066467285,
-                0.027806720733642577,
-                0.027807743072509765,
-                0.02793984031677246,
-                0.02779955291748047,
-                0.027794431686401368,
-                0.02774732780456543,
-                0.027833343505859375,
-                0.02780467224121094,
-                0.027825151443481445,
-                0.02800230407714844,
-                0.02776678466796875,
-                0.02773606491088867,
-                0.02778009605407715,
-                0.027883520126342775,
-                0.027817983627319336,
-                0.027855871200561523,
-                0.027778047561645508,
-                0.027809791564941407,
-                0.0277391357421875,
-                0.027813888549804686,
-                0.02776678466796875,
-                0.02779545593261719,
-                0.027801599502563477,
-                0.027827199935913087,
-                0.02770636749267578,
-                0.027828224182128908,
-                0.02774835205078125,
-                0.027782144546508788,
-                0.027801599502563477,
-                0.027811840057373048,
-                0.02778112030029297,
-                0.02773708724975586,
-                0.027828224182128908,
-                0.02778726387023926,
-                0.027860992431640624,
-                0.02779136085510254,
-                0.02778316879272461,
-                0.02774835205078125,
-                0.027827199935913087,
-                0.027802623748779298,
-                0.02779136085510254,
-                0.02780467224121094,
-                0.02775961685180664,
-                0.027890687942504884,
-                0.02810572814941406,
-                0.027851776123046876,
-                0.02775040054321289,
-                0.027831296920776367,
-                0.027846656799316406,
-                0.027837440490722655,
-                0.027846656799316406,
-                0.02778726387023926,
-                0.027947008132934572,
-                0.027822080612182616,
-                0.027816959381103516,
-                0.027797504425048827,
-                0.027808767318725586,
-                0.02775142478942871,
-                0.02777190399169922,
-                0.02778828811645508,
-                0.027862016677856444,
-                0.027822240829467774,
-                0.02781782341003418,
-                0.027839487075805663,
-                0.02797670364379883,
-                0.027860992431640624,
-                0.027831296920776367,
-                0.02794086456298828,
-                0.027807743072509765,
-                0.027870208740234374,
-                0.027808767318725586,
-                0.028061695098876953,
-                0.027848703384399414,
-                0.02779136085510254,
-                0.027872255325317383,
-                0.02789990425109863,
-                0.027866111755371094,
-                0.027846656799316406,
-                0.027815935134887695,
-                0.027854848861694335,
-                0.027807743072509765,
-                0.027833343505859375,
-                0.02815590476989746,
-                0.027888639450073242,
-                0.027859968185424806,
-                0.027854848861694335,
-                0.027829248428344725,
-                0.027869184494018553,
-                0.027846656799316406,
-                0.02792140769958496,
-                0.028218368530273437,
-                0.027825151443481445,
-                0.027825151443481445,
-                0.027851776123046876,
-                0.027836416244506838,
-                0.027831296920776367,
-                0.027844608306884764,
-                0.027784191131591796,
-                0.027842559814453126,
-                0.027840511322021484,
-                0.027844608306884764,
-                0.02779955291748047,
-                0.027909120559692382,
-                0.027830272674560546,
-                0.027865087509155274,
-                0.027880447387695313,
-                0.027805696487426756,
-                0.027869184494018553,
-                0.027868160247802733,
-                0.027830272674560546,
-                0.027889663696289063,
-                0.02793574333190918,
-                0.02783036804199219,
-                0.027938720703125,
-                0.027871231079101562,
-                0.027856895446777344,
-                0.027852800369262694,
-                0.027863040924072265,
-                0.027862016677856444,
-                0.027913215637207032,
-                0.027872255325317383,
-                0.027871231079101562,
-                0.027827199935913087,
-                0.027820032119750978,
-                0.027820032119750978,
-                0.027813888549804686,
-                0.028193792343139647,
-                0.028027904510498046,
-                0.028009471893310548,
-                0.028003328323364256,
-                0.028076032638549804,
-                0.028169216156005858,
-                0.027986944198608397,
-                0.02797056007385254,
-                0.028003328323364256,
-                0.028045312881469726,
-                0.028087295532226563,
-                0.028251136779785156,
-                0.028099584579467773,
-                0.028044288635253906,
-                0.028075008392333983,
-                0.028051679611206054,
-                0.028056352615356446,
-                0.028071935653686524,
-                0.028041215896606447,
-                0.02797772789001465,
-                0.028070911407470703,
-                0.02794495964050293,
-                0.02795212745666504
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.847169142103404
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 992.038912,
             "max_vram": 15755.902976,
             "max_reserved": 15271.460864,
             "max_allocated": 15114.469888
         },
         "latency": {
             "unit": "s",
+            "mean": 0.03722237179270463,
+            "stdev": 0.00025035189325365263,
             "values": [
+                0.040176830291748046,
+                0.039531520843505856,
+                0.03754905700683594,
+                0.0376258544921875,
+                0.03758694458007812,
+                0.03761663818359375,
+                0.037615615844726565,
+                0.03763916778564453,
+                0.03762073516845703,
+                0.03757689666748047,
+                0.03750707244873047,
+                0.03761971282958984,
+                0.03757875061035156,
+                0.03753062438964844,
+                0.037591041564941405,
+                0.03758899307250976,
+                0.03719987106323242,
+                0.03705241775512695,
+                0.037157886505126955,
+                0.03710566329956055,
+                0.037158943176269534,
+                0.03714777755737304,
+                0.037179393768310545,
+                0.03715071868896484,
+                0.03715686416625977,
+                0.03713536071777344,
+                0.037157886505126955,
+                0.03717324829101563,
+                0.03718860626220703,
+                0.03725516891479492,
+                0.03719270324707031,
+                0.037171199798583986,
+                0.037217281341552735,
+                0.037176319122314457,
+                0.03726335906982422,
+                0.03718860626220703,
+                0.03718147277832031,
+                0.037174270629882815,
+                0.037168128967285156,
+                0.037163009643554686,
+                0.03718889617919922,
+                0.03717555236816406,
+                0.037250049591064455,
+                0.037198848724365234,
+                0.03723775863647461,
+                0.037174270629882815,
+                0.037168128967285156,
+                0.03711078262329102,
+                0.037177345275878904,
+                0.037144577026367184,
+                0.03717529678344727,
+                0.03712102508544922,
+                0.03712204742431641,
+                0.037288959503173826,
+                0.03715708923339844,
+                0.03708419036865234,
+                0.037152767181396484,
+                0.037130241394042966,
+                0.03718143844604492,
+                0.03716403198242187,
+                0.03715891265869141,
+                0.03717529678344727,
+                0.03721318435668945,
+                0.037195999145507815,
+                0.03721932983398438,
+                0.03717027282714844,
+                0.03719094467163086,
+                0.03720806503295898,
+                0.037182464599609374,
+                0.037233665466308595,
+                0.03719987106323242,
+                0.03720499038696289,
+                0.03718041610717773,
+                0.03719680023193359,
+                0.03730739212036133,
+                0.037163009643554686,
+                0.0371599349975586,
+                0.03717324829101563,
+                0.037195777893066405,
+                0.03710771179199219,
+                0.037136383056640625,
                 0.03714355087280274,
+                0.037106719970703125,
+                0.03717324829101563,
+                0.037110881805419924,
+                0.037141502380371096,
+                0.03714771270751953,
+                0.037212158203125,
+                0.03717529678344727,
+                0.03719270324707031,
+                0.037171199798583986,
+                0.03718963241577149,
+                0.037238784790039066,
+                0.037303295135498044,
+                0.03724390411376953,
+                0.0371701774597168,
+                0.037187583923339845,
+                0.03715497589111328,
+                0.03714252853393555,
+                0.03711590576171875,
+                0.03716403198242187,
+                0.03715584182739258,
+                0.037275646209716795,
+                0.037250049591064455,
+                0.03723795318603516,
+                0.03716403198242187,
+                0.03720908737182617,
+                0.03720806503295898,
+                0.03715584182739258,
+                0.03714252853393555,
+                0.03714048004150391,
+                0.03715071868896484,
+                0.03717529678344727,
+                0.03719372940063476,
+                0.037163009643554686,
+                0.03720601654052735,
+                0.03719680023193359,
+                0.03720601654052735,
+                0.03723980712890625,
+                0.03719782257080078,
+                0.03720806503295898,
+                0.03714982223510742,
+                0.03712102508544922,
+                0.03721420669555664,
+                0.03713228988647461,
+                0.03718860626220703,
+                0.03711180877685547,
+                0.03721011352539062,
+                0.03718041610717773,
+                0.037195777893066405,
+                0.037220352172851565,
+                0.03725516891479492,
+                0.03718656158447266,
+                0.03720806503295898,
+                0.037148735046386716,
+                0.0371517448425293,
+                0.037200897216796876,
+                0.03717324829101563,
+                0.037177345275878904,
+                0.037085182189941404,
+                0.037176319122314457,
+                0.037152767181396484,
+                0.03716198348999023,
+                0.03709542465209961,
+                0.03713740921020508,
+                0.03711180877685547,
+                0.03717836761474609,
+                0.03715071868896484,
+                0.037084159851074217,
+                0.03718860626220703,
+                0.03715695953369141,
+                0.03716403198242187,
+                0.037125118255615236,
+                0.037149696350097655,
+                0.037149696350097655,
+                0.03722751998901367,
+                0.03718963241577149,
+                0.03718656158447266,
+                0.03722649765014648,
+                0.037141502380371096,
+                0.03722751998901367,
+                0.03713740921020508,
+                0.037220352172851565,
+                0.03713539123535156,
+                0.037149696350097655,
+                0.03709952163696289,
+                0.037154815673828126,
+                0.03709030532836914,
+                0.0371701774597168,
+                0.037146625518798826,
+                0.037193889617919924,
+                0.03711180877685547,
+                0.037222400665283206,
+                0.03706572723388672,
+                0.037179393768310545,
+                0.037176319122314457,
+                0.037157886505126955,
+                0.0371517448425293,
+                0.03717635345458985,
+                0.03715584182739258,
+                0.03717324829101563,
+                0.037141502380371096,
+                0.03723468780517578,
+                0.03714976119995117,
+                0.037272575378417966,
+                0.03721113586425781,
+                0.03720601654052735,
+                0.0371599349975586,
+                0.03717529678344727,
+                0.037163009643554686,
+                0.03720806503295898,
+                0.03718860626220703,
+                0.03713536071777344,
+                0.03718041610717773,
+                0.03715686416625977,
+                0.037217281341552735,
+                0.03724697494506836,
+                0.03726755142211914,
+                0.03724800109863281,
                 0.0371486701965332,
+                0.037130241394042966,
+                0.0371486701965332,
+                0.0371599349975586,
+                0.03722547149658203,
+                0.03723263931274414,
+                0.03727974319458008,
+                0.0372305908203125,
+                0.037184513092041016,
+                0.03715686416625977,
+                0.03720499038696289,
+                0.03713228988647461,
+                0.037171199798583986,
+                0.037146625518798826,
+                0.03718963241577149,
+                0.03716505432128906,
+                0.03717756652832031,
+                0.03726131057739258,
+                0.03719372940063476,
+                0.03723993682861328,
+                0.03720294570922852,
+                0.03721318435668945,
+                0.037152767181396484,
+                0.03722137451171875,
+                0.03713945770263672,
+                0.03720601654052735,
+                0.03715584182739258,
+                0.03726233673095703,
+                0.03723980712890625,
+                0.03721830368041992,
+                0.037217281341552735,
+                0.0372490234375,
+                0.037147647857666014,
+                0.03722655868530273,
+                0.037200897216796876,
+                0.03723795318603516,
+                0.03720191955566406,
+                0.03725632095336914,
+                0.037220352172851565,
+                0.03719702529907227,
+                0.037138431549072266,
+                0.037179393768310545,
+                0.03715071868896484,
+                0.03718348693847656,
+                0.03713433456420898,
+                0.03723980712890625,
+                0.03719270324707031,
+                0.037200927734375,
+                0.037179393768310545,
+                0.03719475173950195,
+                0.037219615936279295,
+                0.03716505432128906,
+                0.037163009643554686,
+                0.03716505432128906,
+                0.03721113586425781,
+                0.037157886505126955,
+                0.037269504547119144,
+                0.03717836761474609,
+                0.037220352172851565,
+                0.03722137451171875,
+                0.037176319122314457,
+                0.03721932983398438,
+                0.03715584182739258,
+                0.03717222213745117,
+                0.037169151306152344,
+                0.03726150512695312,
+                0.03722751998901367,
+                0.03723468780517578,
+                0.03715071868896484,
+                0.03720191955566406
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 6877.584304022629
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 992.038912,
             "max_vram": 15925.772288,
             "max_reserved": 15441.330176,
             "max_allocated": 15238.453248
         },
         "latency": {
             "unit": "s",
+            "mean": 7.4603218002319345,
+            "stdev": 0.0020198326110834053,
             "values": [
+                7.4583019676208515,
+                7.462341632843018
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 34.18083117970492
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.02925616392247818,
+            "stdev": 0.00038938498684848596,
             "values": [
+                0.03159756851196289,
+                0.030905344009399413,
+                0.02921779251098633,
+                0.02914201545715332,
+                0.029138944625854493,
+                0.02913689613342285,
+                0.02913484764099121,
+                0.029174783706665038,
+                0.029240320205688477,
+                0.029087743759155273,
+                0.02919321632385254,
+                0.029154304504394532,
+                0.02911027145385742,
+                0.029231103897094726,
+                0.029099008560180665,
+                0.02913484764099121,
+                0.029155328750610353,
+                0.02913689613342285,
+                0.02915225601196289,
+                0.029146112442016602,
+                0.02911846351623535,
+                0.029139968872070314,
+                0.029197311401367186,
+                0.02913484764099121,
+                0.029146112442016602,
+                0.02916659164428711,
+                0.029090816497802735,
+                0.029173759460449217,
+                0.029097984313964844,
+                0.02916147232055664,
+                0.02917683219909668,
+                0.029162496566772462,
+                0.02915328025817871,
+                0.029121536254882813,
+                0.029130752563476563,
+                0.029109247207641603,
+                0.02918604850769043,
+                0.029139968872070314,
+                0.029074432373046875,
+                0.02915839958190918,
+                0.029081600189208984,
+                0.029221887588500976,
                 0.029270015716552734,
+                0.02913382339477539,
+                0.029273088455200196,
+                0.029260799407958983,
+                0.029097984313964844,
+                0.029155328750610353,
+                0.029174783706665038,
+                0.02911948776245117,
+                0.029116416931152345,
+                0.029157375335693358,
+                0.029124607086181642,
+                0.029086719512939452,
+                0.029146112442016602,
+                0.029108224868774416,
+                0.02918707275390625,
+                0.029197311401367186,
+                0.029068288803100587,
+                0.0292044792175293,
+                0.029206527709960937,
+                0.029132799148559572,
+                0.02912563133239746,
+                0.029087743759155273,
+                0.029129728317260743,
+                0.029163520812988283,
+                0.02916761589050293,
+                0.02917683219909668,
+                0.0291778564453125,
+                0.029243392944335936,
+                0.02922598457336426,
+                0.02912665557861328,
+                0.029101055145263673,
+                0.029165567398071288,
+                0.02910207939147949,
+                0.02913587188720703,
+                0.02913587188720703,
+                0.029220863342285155,
+                0.029173759460449217,
+                0.0291727352142334,
+                0.029223936080932617,
+                0.02920857620239258,
+                0.02918502426147461,
+                0.029258752822875978,
+                0.0291409912109375,
+                0.029945856094360353,
+                0.030311424255371092,
                 0.029247488021850586,
+                0.029163520812988283,
+                0.02918502426147461,
+                0.029207551956176758,
+                0.02938572883605957,
+                0.02919219207763672,
+                0.029130752563476563,
+                0.0292044792175293,
+                0.029277183532714843,
+                0.02921881675720215,
+                0.02916966438293457,
+                0.029255680084228516,
+                0.029237247467041014,
+                0.02916147232055664,
+                0.02921897506713867,
+                0.029227872848510743,
+                0.029223936080932617,
+                0.029285375595092773,
+                0.029277183532714843,
+                0.02918502426147461,
+                0.029205503463745116,
+                0.029315071105957033,
+                0.02923520088195801,
+                0.02922700881958008,
+                0.029207551956176758,
+                0.029246463775634765,
+                0.029199359893798828,
+                0.029288448333740235,
+                0.029297664642333986,
+                0.029303808212280274,
+                0.029280256271362305,
+                0.02922598457336426,
+                0.029197311401367186,
+                0.029281280517578126,
+                0.029232128143310547,
+                0.029312000274658204,
+                0.029286399841308593,
+                0.029283327102661134,
+                0.029244415283203123,
+                0.029239295959472656,
+                0.02921881675720215,
+                0.029216768264770508,
+                0.029216768264770508,
+                0.029211648941040037,
+                0.029191167831420898,
+                0.029228031158447267,
+                0.029298688888549803,
+                0.029213695526123046,
+                0.02916659164428711,
+                0.02918502426147461,
+                0.0293253116607666,
+                0.029191167831420898,
+                0.029189119338989256,
+                0.02919526481628418,
+                0.02922598457336426,
                 0.029257728576660157,
+                0.02919219207763672,
+                0.029246463775634765,
+                0.02918809509277344,
+                0.02916044807434082,
+                0.029215744018554687,
+                0.029224960327148438,
+                0.029222911834716796,
+                0.029179904937744142,
+                0.02922598457336426,
+                0.02922598457336426,
+                0.02920345687866211,
+                0.029240320205688477,
+                0.029279232025146484,
+                0.029245439529418944,
+                0.02920857620239258,
+                0.029265920639038087,
+                0.02924236869812012,
+                0.029263872146606446,
+                0.029282304763793947,
+                0.029256704330444337,
+                0.029294591903686523,
+                0.029271039962768555,
+                0.029273088455200196,
+                0.029262847900390625,
+                0.02924236869812012,
+                0.029297664642333986,
+                0.029256704330444337,
+                0.029306880950927733,
+                0.029298688888549803,
+                0.029262847900390625,
+                0.029280256271362305,
+                0.029216768264770508,
+                0.02925056076049805,
+                0.029230079650878905,
+                0.029212671279907225,
+                0.029223936080932617,
+                0.029272064208984375,
+                0.029251583099365236,
+                0.029254655838012695,
+                0.02933145523071289,
+                0.029228031158447267,
+                0.029216768264770508,
+                0.02920140838623047,
+                0.029213695526123046,
+                0.029243392944335936,
+                0.029236223220825194,
+                0.029260799407958983,
+                0.029285375595092773,
+                0.02931711959838867,
+                0.029282304763793947,
+                0.029224960327148438,
+                0.029237247467041014,
+                0.02916966438293457,
+                0.029237247467041014,
+                0.02919219207763672,
+                0.029258752822875978,
+                0.029271039962768555,
+                0.029293567657470702,
+                0.029283327102661134,
+                0.02923417663574219,
+                0.029219839096069337,
+                0.029182975769042968,
+                0.029254655838012695,
+                0.029282304763793947,
+                0.02935603141784668,
+                0.02929971122741699,
+                0.029266944885253908,
+                0.029273088455200196,
+                0.02921779251098633,
+                0.029277183532714843,
+                0.02925056076049805,
+                0.029314048767089845,
+                0.02934988784790039,
+                0.02934783935546875,
+                0.0293570556640625,
+                0.029336576461791993,
+                0.029258752822875978,
+                0.029329408645629884,
+                0.029298688888549803,
+                0.029295616149902344,
+                0.02935910415649414,
+                0.029288448333740235,
+                0.029277183532714843,
+                0.02930892753601074,
+                0.02930073547363281,
+                0.029271039962768555,
+                0.029335552215576172,
+                0.02936422348022461,
+                0.02930073547363281,
+                0.02939187240600586,
+                0.029305856704711915,
+                0.02933964729309082,
+                0.029279232025146484,
+                0.029321216583251954,
+                0.02933350372314453,
+                0.029414400100708008,
+                0.029464576721191408,
+                0.029285375595092773,
+                0.02938982391357422,
+                0.029230079650878905,
+                0.029386751174926756,
+                0.029212671279907225,
+                0.029281280517578126,
+                0.029345792770385744,
+                0.029270015716552734,
+                0.029219839096069337,
+                0.029327360153198243,
+                0.029329408645629884,
+                0.029258752822875978,
+                0.029266944885253908,
+                0.029223936080932617,
+                0.02924236869812012,
+                0.029129728317260743,
+                0.029117439270019533,
+                0.02910207939147949,
+                0.029092863082885743,
+                0.029165567398071288,
+                0.029123584747314454,
+                0.02916761589050293,
+                0.02909388732910156,
+                0.029181951522827147,
+                0.02909388732910156,
+                0.029207551956176758,
+                0.029124607086181642,
+                0.029130752563476563,
+                0.029212671279907225,
+                0.029105152130126953,
+                0.029147136688232423,
+                0.02911129570007324,
+                0.029089792251586914,
+                0.02915225601196289,
+                0.029131776809692384,
+                0.0291727352142334,
+                0.02920243263244629,
+                0.02920345687866211,
+                0.029061119079589845,
+                0.029199359893798828,
+                0.02923520088195801,
+                0.029088768005371093,
+                0.029130752563476563,
+                0.029057024002075195,
+                0.02916966438293457,
+                0.029086719512939452,
+                0.029089792251586914,
+                0.029149183273315428,
+                0.029107200622558595,
+                0.029050880432128907,
+                0.029116416931152345,
+                0.029055999755859374,
+                0.029106176376342774,
+                0.02915225601196289,
+                0.02916044807434082,
+                0.02912665557861328,
+                0.029173759460449217,
+                0.02917683219909668,
+                0.029190143585205077,
+                0.02918707275390625,
+                0.029245439529418944,
+                0.029163520812988283,
+                0.02919424057006836,
+                0.02915225601196289,
+                0.0291778564453125,
+                0.029121536254882813,
+                0.02911027145385742,
+                0.029149183273315428,
+                0.02909388732910156,
+                0.029131776809692384,
+                0.029138944625854493,
+                0.0292096004486084,
+                0.02923520088195801,
+                0.029258752822875978,
+                0.02917068862915039,
+                0.029146112442016602,
+                0.029239295959472656,
+                0.029114368438720704,
+                0.029231103897094726,
+                0.02917683219909668,
+                0.029139968872070314,
+                0.02923520088195801,
+                0.0292096004486084,
+                0.02916864013671875,
+                0.02916659164428711,
+                0.029231103897094726,
+                0.02919424057006836,
+                0.02915020751953125,
+                0.02917683219909668,
+                0.029155328750610353,
+                0.02920140838623047,
+                0.029137920379638672,
+                0.02919526481628418,
+                0.029236223220825194,
+                0.029243392944335936,
+                0.02920140838623047,
+                0.029197311401367186,
+                0.029189119338989256,
+                0.02918502426147461,
+                0.02913689613342285,
+                0.029211648941040037,
+                0.029165567398071288,
+                0.029205503463745116,
+                0.029249536514282228,
+                0.029208671569824218,
+                0.029212575912475586,
+                0.02918400001525879,
+                0.029191167831420898,
+                0.029163520812988283,
+                0.02917580795288086,
+                0.02917068862915039,
+                0.029146112442016602,
+                0.029189119338989256,
+                0.029211711883544923,
+                0.029193151473999025,
+                0.029280256271362305,
+                0.029236223220825194,
+                0.029321216583251954,
+                0.02925056076049805,
+                0.02921881675720215,
+                0.02920038414001465,
+                0.029213695526123046,
+                0.029276159286499022,
+                0.029221887588500976,
+                0.029277183532714843,
+                0.029197311401367186,
+                0.029237247467041014,
+                0.029222911834716796,
+                0.029215744018554687,
+                0.029219839096069337,
+                0.029224960327148438,
+                0.029206527709960937,
+                0.029132799148559572,
+                0.029219839096069337,
+                0.02917683219909668,
+                0.029260799407958983,
+                0.02917888069152832,
+                0.02921881675720215,
+                0.029191167831420898,
+                0.029171712875366212,
+                0.029213695526123046,
+                0.029207551956176758,
+                0.029212671279907225,
+                0.02916864013671875,
+                0.029255680084228516,
+                0.029238271713256835,
+                0.029224960327148438,
+                0.029181951522827147,
+                0.02917888069152832,
+                0.029162496566772462,
+                0.029222911834716796,
+                0.029180927276611326,
+                0.02916147232055664,
+                0.02920345687866211,
+                0.029267967224121092,
+                0.029248512268066407,
+                0.029179904937744142,
+                0.02918707275390625,
+                0.029236223220825194,
+                0.029165567398071288,
+                0.029173759460449217,
+                0.029272064208984375,
+                0.029284351348876952,
+                0.0291276798248291,
+                0.02933247947692871,
+                0.02919526481628418,
+                0.029256704330444337,
+                0.029220863342285155,
+                0.02915225601196289,
+                0.02915839958190918,
+                0.0292044792175293,
+                0.029225055694580077,
+                0.029190048217773438,
+                0.029343744277954102,
+                0.02923417663574219,
+                0.029214719772338867,
+                0.029245439529418944,
+                0.029327360153198243,
+                0.02918400001525879,
+                0.029285375595092773,
+                0.029253631591796874,
+                0.02920857620239258,
+                0.029284351348876952,
+                0.029232128143310547,
+                0.02922700881958008,
+                0.029277183532714843,
+                0.029253631591796874,
+                0.029211648941040037,
+                0.029321216583251954,
+                0.029264896392822266,
+                0.029297664642333986,
+                0.029245439529418944,
+                0.02914406394958496,
+                0.029261823654174804,
+                0.029267967224121092,
+                0.029253631591796874,
+                0.029290496826171877,
+                0.029301759719848632,
+                0.029313024520874024,
+                0.029207551956176758,
+                0.029198335647583007,
+                0.02929254341125488,
+                0.029278207778930664,
+                0.02938368034362793,
+                0.029275136947631834,
+                0.02936627197265625,
+                0.029214719772338867,
+                0.02920140838623047,
+                0.029271039962768555,
+                0.029280256271362305,
+                0.0293304328918457,
+                0.02932633590698242,
+                0.029260799407958983,
+                0.029260799407958983,
+                0.029276159286499022,
+                0.029270015716552734,
+                0.02931609535217285,
+                0.029232128143310547,
+                0.02929254341125488,
+                0.029278207778930664,
+                0.029273088455200196,
+                0.029248512268066407,
+                0.029212671279907225,
+                0.029295616149902344,
+                0.029320192337036134,
+                0.029260799407958983,
+                0.0293703670501709,
+                0.0293253116607666,
+                0.02934783935546875,
+                0.02929254341125488,
+                0.02930073547363281,
+                0.029293567657470702,
+                0.0293253116607666,
+                0.029284351348876952,
+                0.029294591903686523,
+                0.02929254341125488,
+                0.029289472579956056,
+                0.029290496826171877,
+                0.029262847900390625,
+                0.02929254341125488,
+                0.029266944885253908,
+                0.029301759719848632,
+                0.02935807991027832,
+                0.029293567657470702,
+                0.029241344451904298,
+                0.029260799407958983,
+                0.029352960586547853,
+                0.029273088455200196,
+                0.029377536773681642,
+                0.029322240829467775,
+                0.02919526481628418,
+                0.029295616149902344,
+                0.029371391296386717,
+                0.029221887588500976,
+                0.029282304763793947,
+                0.02921881675720215,
+                0.029233152389526368,
+                0.029301759719848632,
+                0.029296640396118165,
+                0.02930790328979492,
+                0.029312000274658204,
+                0.02933452796936035,
+                0.029429759979248047,
+                0.029321216583251954,
+                0.029430784225463868,
+                0.029428735733032226,
+                0.030038015365600586,
+                0.03061350440979004,
+                0.030520320892333985,
+                0.03702169418334961
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 34.18083117970491
         },
         "energy": null,
         "efficiency": null