kreas
/

FA_benchmarks

Model card Files Files and versions Community

kreas commited on Mar 14

Commit

b51db5a

•

1 Parent(s): ebdb2e5

Upload /phi-2/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

phi-2/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +738 -738

phi-2/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,292 +2,292 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 964.23936,
             "max_vram": 2585.788416,
             "max_reserved": 2101.346304,
             "max_allocated": 2014.912
         },
         "latency": {
             "unit": "s",
-            "mean": 0.03698565444383237,
-            "stdev": 0.0004510177520171469,
             "values": [
-                0.040046463012695314,
-                0.03841535949707031,
-                0.03833446502685547,
-                0.037953536987304685,
-                0.03789823913574219,
-                0.03812454223632813,
-                0.03973961639404297,
-                0.041866241455078126,
-                0.03862220764160156,
-                0.03701657485961914,
-                0.03690188980102539,
-                0.036831230163574216,
-                0.03707699203491211,
-                0.03682304000854492,
-                0.03681280136108398,
-                0.03687321472167969,
-                0.03697868728637695,
-                0.036898815155029296,
-                0.03687526321411133,
-                0.03678617477416992,
-                0.03687526321411133,
-                0.03685990524291992,
-                0.036908031463623044,
-                0.036738048553466796,
-                0.03678515243530273,
-                0.036841472625732424,
-                0.03693875122070313,
-                0.036853759765625,
-                0.03682611083984375,
-                0.03723468780517578,
-                0.036794654846191405,
-                0.0368721923828125,
-                0.03683327865600586,
-                0.03692748641967773,
-                0.03684044647216797,
-                0.03696640014648438,
-                0.036931583404541016,
-                0.03688959884643555,
-                0.03674521636962891,
-                0.03685068893432617,
-                0.03673212814331055,
-                0.036790271759033204,
-                0.03718860626220703,
-                0.03680972671508789,
-                0.03685068893432617,
-                0.03680255889892578,
-                0.03683647918701172,
                 0.03683225631713867,
-                0.03686502456665039,
-                0.03682099151611328,
-                0.03683430480957031,
                 0.036918270111083985,
                 0.036953086853027346,
-                0.036837406158447265,
-                0.03681792068481445,
-                0.03687936019897461,
-                0.03683327865600586,
                 0.03678003311157227,
-                0.036736000061035154,
                 0.03679641723632812,
-                0.03672780990600586,
-                0.03685683059692383,
-                0.036743167877197266,
-                0.03676979064941406,
-                0.036760574340820314,
-                0.036775070190429686,
-                0.03673088073730469,
-                0.03686195373535156,
-                0.03670540618896485,
-                0.0368353271484375,
-                0.03673395156860351,
-                0.036822017669677735,
-                0.03672694396972656,
-                0.03673395156860351,
-                0.036770816802978515,
-                0.03678003311157227,
-                0.036746559143066404,
                 0.036915199279785156,
-                0.03684966278076172,
-                0.03700326538085937,
-                0.03687334442138672,
-                0.036746463775634765,
-                0.03687321472167969,
-                0.036926464080810545,
-                0.03707392120361328,
-                0.03678822326660156,
-                0.03692748641967773,
-                0.03681792068481445,
-                0.03684249496459961,
-                0.03679743957519531,
-                0.03690598297119141,
-                0.03681894302368164,
-                0.03674828720092774,
-                0.03670016098022461,
-                0.03678335952758789,
-                0.036751361846923826,
-                0.03678515243530273,
-                0.037233665466308595,
-                0.03681792068481445,
-                0.03680665588378906,
-                0.03683942413330078,
-                0.03688959884643555,
-                0.0369758415222168,
-                0.03681587219238281,
                 0.03684659194946289,
-                0.03684044647216797,
-                0.03679743957519531,
-                0.03693772888183594,
-                0.03684249496459961,
-                0.03684044647216797,
-                0.03676691055297852,
-                0.036896766662597655,
-                0.03690291213989258,
-                0.036907009124755856,
-                0.03685068893432617,
-                0.036928512573242187,
-                0.03695513534545898,
-                0.03693056106567383,
-                0.036841472625732424,
                 0.037010433197021485,
-                0.037043201446533204,
-                0.036972545623779295,
-                0.03690086364746094,
-                0.03720294570922852,
                 0.036942848205566405,
-                0.036877311706542966,
-                0.03702783966064453,
-                0.03688044738769531,
-                0.03695206451416016,
-                0.03707187271118164,
-                0.03698175811767578,
-                0.036939777374267575,
-                0.036955455780029296,
-                0.03700121688842774,
-                0.03704729461669922,
-                0.036913150787353514,
-                0.03693363189697266,
-                0.03696230316162109,
-                0.03743743896484375,
-                0.0369879035949707,
-                0.03694387054443359,
-                0.03690496063232422,
-                0.03689471817016601,
-                0.03695206451416016,
-                0.03691417694091797,
-                0.03701760101318359,
-                0.03696537780761719,
-                0.03693260955810547,
-                0.036929534912109374,
-                0.03696230316162109,
-                0.03689779281616211,
-                0.03729126358032227,
-                0.03690496063232422,
-                0.03697049713134765,
-                0.03683737564086914,
-                0.03694079971313476,
-                0.03696246337890625,
-                0.03694899368286133,
                 0.03688140869140625,
-                0.03694287872314453,
                 0.036890625,
-                0.03692134475708008,
-                0.0368721923828125,
-                0.03693875122070313,
-                0.03711795043945312,
-                0.036944896697998046,
-                0.03687321472167969,
-                0.03695513534545898,
-                0.03684966278076172,
                 0.03696640014648438,
-                0.03687936019897461,
-                0.037009471893310546,
-                0.0369541130065918,
                 0.03693670272827149,
-                0.03695206451416016,
-                0.03685171127319336,
-                0.03684249496459961,
                 0.03687628936767578,
-                0.03695743942260742,
-                0.036877311706542966,
-                0.036888671875,
-                0.03690496063232422,
-                0.03697356796264648,
-                0.036959232330322264,
                 0.036880382537841795,
-                0.036951038360595705,
-                0.03682611083984375,
-                0.03709235382080078,
-                0.03683635330200195,
-                0.036805633544921876,
-                0.03687936019897461,
-                0.03708313751220703,
-                0.036980735778808595,
-                0.03728998565673828,
-                0.03689984130859375,
-                0.03685068893432617,
-                0.036961280822753906,
-                0.03691417694091797,
-                0.036923393249511716,
-                0.036951038360595705,
-                0.03691622543334961,
-                0.036893695831298826,
-                0.036907009124755856,
-                0.03694825744628906,
-                0.036873279571533205,
-                0.03694182586669922,
-                0.037400577545166014,
-                0.036947967529296875,
-                0.036910079956054685,
-                0.03704627227783203,
-                0.03706675338745117,
-                0.03695820617675781,
-                0.036944896697998046,
-                0.0369541130065918,
-                0.03698387145996094,
-                0.03704422378540039,
-                0.03685068893432617,
-                0.03691017532348633,
-                0.036822017669677735,
-                0.0369879035949707,
-                0.03681792068481445,
-                0.036790271759033204,
-                0.03691417694091797,
-                0.037116928100585936,
-                0.03694387054443359,
-                0.03694694519042969,
                 0.036945919036865234,
-                0.03712847900390625,
-                0.036994049072265625,
-                0.0370882568359375,
-                0.03686406326293945,
-                0.037015552520751956,
-                0.03693056106567383,
-                0.03708848190307617,
-                0.036890625,
-                0.03697987365722656,
-                0.036896766662597655,
-                0.036969470977783206,
-                0.03686940765380859,
-                0.03697151947021484,
                 0.03685990524291992,
-                0.03694204711914063,
-                0.03697049713134765,
-                0.036985855102539066,
-                0.03686195373535156,
-                0.0369541130065918,
-                0.036898815155029296,
-                0.03737702560424805,
-                0.03688243103027344,
-                0.037043201446533204,
-                0.037043201446533204,
-                0.03700838470458984,
-                0.03710275268554687,
-                0.036951038360595705,
                 0.03696537780761719,
-                0.036980735778808595,
-                0.03701862335205078,
-                0.036994049072265625,
                 0.03691622543334961,
-                0.03693670272827149,
-                0.03731455993652344,
-                0.036983806610107424,
-                0.03691219329833984,
                 0.036915199279785156,
-                0.03697663879394531,
-                0.03694387054443359,
                 0.03688243103027344,
-                0.037026817321777344,
                 0.036947967529296875,
-                0.036928512573242187,
-                0.036982784271240236
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 6921.6025469758815
         },
         "energy": null,
         "efficiency": null
@@ -295,23 +295,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 964.23936,
             "max_vram": 2858.418176,
             "max_reserved": 2373.976064,
             "max_allocated": 2326.085632
         },
         "latency": {
             "unit": "s",
-            "mean": 7.248771577835086,
-            "stdev": 0.04500223922729729,
             "values": [
-                7.203769338607789,
-                7.293773817062384
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.17837433031076
         },
         "energy": null,
         "efficiency": null
@@ -320,524 +320,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.028426555207196384,
-            "stdev": 0.0007068028114393512,
             "values": [
-                0.028911615371704103,
-                0.028435455322265626,
-                0.028400640487670898,
-                0.028308479309082032,
                 0.02835763168334961,
-                0.0283504638671875,
-                0.028325887680053712,
                 0.02835353660583496,
-                0.028296192169189452,
-                0.02830438423156738,
-                0.028318719863891603,
-                0.028263423919677736,
-                0.02831667137145996,
-                0.02830745506286621,
-                0.028244991302490235,
-                0.028421119689941408,
-                0.02831564712524414,
-                0.028310527801513673,
-                0.028403711318969727,
-                0.02833510398864746,
-                0.028229631423950196,
-                0.028314624786376953,
-                0.028265632629394532,
-                0.028312416076660157,
-                0.028291072845458985,
                 0.0283371524810791,
-                0.028277759552001954,
-                0.02832793617248535,
                 0.02834124755859375,
-                0.028399616241455077,
                 0.028309503555297853,
-                0.02833510398864746,
-                0.028483583450317384,
-                0.028447744369506835,
-                0.028280832290649413,
-                0.028313600540161132,
-                0.02834841537475586,
-                0.02833919906616211,
-                0.02833510398864746,
-                0.028647424697875977,
-                0.028290111541748045,
-                0.02844051170349121,
-                0.028253183364868165,
-                0.02838425636291504,
                 0.028416000366210937,
-                0.028358655929565428,
                 0.02830335998535156,
-                0.028279808044433592,
-                0.02836582374572754,
-                0.028308479309082032,
-                0.028267520904541016,
-                0.028564479827880858,
-                0.028278783798217775,
-                0.028392448425292968,
-                0.028222463607788087,
-                0.02819584083557129,
-                0.02854092788696289,
-                0.02813132858276367,
-                0.028096511840820314,
-                0.0281395206451416,
-                0.028073984146118162,
-                0.028189695358276368,
-                0.028215295791625978,
-                0.028226560592651367,
-                0.028237823486328126,
-                0.02812620735168457,
-                0.028089376449584962,
-                0.02814460754394531,
-                0.02809753608703613,
-                0.028185600280761718,
-                0.028219392776489258,
-                0.02817433547973633,
-                0.028188703536987304,
-                0.028273632049560547,
-                0.028203008651733398,
-                0.028272640228271483,
-                0.02832076835632324,
                 0.028286975860595705,
-                0.028462080001831053,
-                0.028271615982055662,
-                0.028267520904541016,
-                0.02813542366027832,
-                0.028083200454711913,
-                0.02817433547973633,
-                0.02813132858276367,
-                0.028169216156005858,
-                0.02814156723022461,
-                0.02831667137145996,
-                0.028165119171142578,
-                0.028230655670166017,
-                0.028100608825683594,
-                0.028112895965576173,
-                0.02818252754211426,
-                0.028258304595947265,
-                0.02819584083557129,
-                0.028207103729248048,
-                0.028217344284057616,
-                0.02815488052368164,
-                0.028200960159301756,
-                0.02815897560119629,
-                0.028227584838867188,
-                0.028274688720703125,
-                0.028177408218383788,
-                0.028203008651733398,
-                0.02821855926513672,
-                0.028306240081787108,
-                0.028197887420654297,
-                0.028224512100219725,
-                0.028236799240112305,
-                0.028216320037841795,
                 0.02862387275695801,
-                0.028222463607788087,
-                0.028224512100219725,
-                0.02840575981140137,
-                0.028248064041137694,
-                0.028196863174438477,
-                0.02814668846130371,
-                0.0281343994140625,
-                0.028056575775146485,
-                0.028081151962280275,
-                0.02815283203125,
-                0.02815795135498047,
-                0.02813337516784668,
-                0.02815999984741211,
-                0.0281395206451416,
-                0.02815795135498047,
-                0.028074047088623048,
-                0.028581823348999023,
-                0.028071935653686524,
-                0.028057600021362306,
-                0.027998207092285156,
-                0.028102655410766602,
-                0.028043264389038085,
-                0.02810572814941406,
-                0.028116992950439453,
-                0.02813849639892578,
-                0.02798080062866211,
-                0.028047359466552735,
-                0.0279736328125,
-                0.02817638397216797,
-                0.028185600280761718,
-                0.02818662452697754,
-                0.02817638397216797,
-                0.028223487854003908,
-                0.02813849639892578,
-                0.028263423919677736,
-                0.028211200714111328,
-                0.028252159118652344,
-                0.02816307258605957,
-                0.028212223052978515,
-                0.0285665283203125,
-                0.028265472412109374,
-                0.02815692710876465,
-                0.028211200714111328,
-                0.028277759552001954,
-                0.028258304595947265,
-                0.028265472412109374,
-                0.028255231857299806,
-                0.028119136810302734,
-                0.028214176177978514,
-                0.028092416763305664,
-                0.028078079223632812,
-                0.028099584579467773,
-                0.02812518310546875,
-                0.02818355178833008,
-                0.02814259147644043,
-                0.02812518310546875,
-                0.028203008651733398,
-                0.02836479949951172,
-                0.02810163116455078,
-                0.02812928009033203,
-                0.02819174385070801,
-                0.02814361572265625,
-                0.028270591735839845,
-                0.02812313652038574,
-                0.02814771270751953,
-                0.02820812797546387,
-                0.028250112533569335,
-                0.028112895965576173,
-                0.028212223052978515,
-                0.028161024093627928,
-                0.028227584838867188,
-                0.028719104766845704,
-                0.028396543502807618,
-                0.028242944717407226,
-                0.028296192169189452,
-                0.028049407958984376,
-                0.028107776641845703,
-                0.02796236801147461,
-                0.028043264389038085,
-                0.028154943466186525,
-                0.02812614440917969,
-                0.028103679656982423,
-                0.02835251235961914,
-                0.02819071960449219,
-                0.028238847732543947,
-                0.02814793586730957,
-                0.02830723190307617,
-                0.028259328842163086,
-                0.028862464904785157,
-                0.028201984405517577,
-                0.028250112533569335,
-                0.02815692710876465,
-                0.028301311492919923,
-                0.028246015548706056,
-                0.028302335739135744,
-                0.028301536560058595,
-                0.028325664520263673,
-                0.028278783798217775,
-                0.028412927627563478,
-                0.02832076835632324,
-                0.028400640487670898,
                 0.028233728408813476,
-                0.02836992073059082,
-                0.028254207611083985,
-                0.02835763168334961,
-                0.02814771270751953,
-                0.02815795135498047,
-                0.028282880783081055,
-                0.028290048599243164,
                 0.028263423919677736,
-                0.028271615982055662,
-                0.028233728408813476,
-                0.028288000106811522,
-                0.028236799240112305,
-                0.028411903381347657,
-                0.028379135131835938,
-                0.028399616241455077,
-                0.028203008651733398,
-                0.02833919906616211,
-                0.02832691192626953,
-                0.028264448165893553,
-                0.028293119430541993,
-                0.028215295791625978,
-                0.028255231857299806,
-                0.028318719863891603,
-                0.028249088287353515,
-                0.02834943962097168,
-                0.02832486343383789,
-                0.02853273582458496,
                 0.02834534454345703,
-                0.028299264907836914,
-                0.028269567489624024,
-                0.028277759552001954,
-                0.02836992073059082,
-                0.028308479309082032,
-                0.028293119430541993,
-                0.028290048599243164,
-                0.02824608039855957,
-                0.028138431549072265,
-                0.028049407958984376,
-                0.02816819190979004,
-                0.02815692710876465,
-                0.02853785514831543,
-                0.028103679656982423,
                 0.028387327194213868,
-                0.028417024612426758,
                 0.02836787223815918,
-                0.028466175079345703,
-                0.028445695877075194,
-                0.028523519515991212,
-                0.028376256942749024,
-                0.028446527481079103,
-                0.028334079742431642,
                 0.028413951873779295,
-                0.028468223571777345,
-                0.028483583450317384,
-                0.02835148811340332,
-                0.02854604721069336,
-                0.028906496047973632,
-                0.028411903381347657,
-                0.028536832809448243,
-                0.028470272064208983,
                 0.028387327194213868,
-                0.02838835144042969,
-                0.028246015548706056,
-                0.028404735565185548,
-                0.028262399673461915,
-                0.02834124755859375,
-                0.02837196731567383,
-                0.02851020812988281,
-                0.028463104248046874,
-                0.028523519515991212,
-                0.028493824005126952,
-                0.028448768615722656,
-                0.028403711318969727,
-                0.02852659225463867,
                 0.028461055755615236,
                 0.02840575981140137,
-                0.02832486343383789,
-                0.02831974411010742,
-                0.028483583450317384,
-                0.02842624092102051,
-                0.028267520904541016,
-                0.028464128494262695,
-                0.02840166473388672,
-                0.028398591995239256,
-                0.028468223571777345,
-                0.028309503555297853,
-                0.02837299156188965,
-                0.028447744369506835,
-                0.02853376007080078,
-                0.028483583450317384,
-                0.02838937568664551,
                 0.028440576553344726,
-                0.028499967575073244,
-                0.028471296310424804,
-                0.028471296310424804,
-                0.02856857681274414,
-                0.028427263259887696,
-                0.02835251235961914,
-                0.028405855178833008,
-                0.028382112503051758,
-                0.02840575981140137,
-                0.02835558319091797,
-                0.02834739112854004,
-                0.02835763168334961,
-                0.02837196731567383,
-                0.028438528060913085,
-                0.028485631942749022,
-                0.02854092788696289,
-                0.02839449691772461,
-                0.028420095443725587,
-                0.028702720642089844,
-                0.028379135131835938,
                 0.02837606430053711,
-                0.028411903381347657,
-                0.028253183364868165,
                 0.02832793617248535,
-                0.028269567489624024,
-                0.028448768615722656,
-                0.028379135131835938,
-                0.028407808303833007,
-                0.028234752655029297,
-                0.02836070442199707,
-                0.028237823486328126,
-                0.028314624786376953,
-                0.028256256103515624,
-                0.028459007263183594,
-                0.028318719863891603,
-                0.02896691131591797,
-                0.02837606430053711,
-                0.028451839447021485,
                 0.02839756774902344,
-                0.028398591995239256,
-                0.028196863174438477,
-                0.028280832290649413,
-                0.02815283203125,
-                0.028207103729248048,
-                0.028235776901245117,
-                0.02840985679626465,
-                0.02819891166687012,
-                0.028259328842163086,
-                0.02813542366027832,
-                0.028144639968872072,
-                0.02816819190979004,
-                0.028212223052978515,
-                0.02815488052368164,
-                0.028284927368164063,
-                0.028212223052978515,
-                0.02832076835632324,
-                0.028273664474487304,
-                0.028285951614379884,
-                0.028275711059570312,
-                0.02833612823486328,
-                0.028241920471191406,
-                0.028375040054321288,
-                0.028286975860595705,
-                0.02834432029724121,
                 0.028305408477783203,
-                0.028256256103515624,
-                0.028310527801513673,
-                0.028410879135131836,
-                0.02834230422973633,
-                0.02840982437133789,
-                0.028298240661621094,
-                0.028252159118652344,
-                0.028215295791625978,
-                0.02833305549621582,
-                0.028300287246704102,
-                0.028396543502807618,
-                0.028319839477539063,
-                0.0283155517578125,
-                0.028309503555297853,
-                0.02834943962097168,
-                0.02834124755859375,
-                0.028387327194213868,
-                0.028267520904541016,
-                0.028297216415405273,
-                0.028283903121948242,
-                0.028301311492919923,
-                0.028329984664916992,
-                0.028281856536865234,
-                0.028274688720703125,
-                0.02873855972290039,
-                0.028413951873779295,
-                0.02839756774902344,
-                0.028281856536865234,
-                0.028215295791625978,
-                0.02836992073059082,
-                0.02833510398864746,
-                0.028289024353027343,
-                0.02857369613647461,
-                0.03272192001342773,
-                0.033925121307373046,
-                0.033454078674316406,
-                0.032860160827636715,
-                0.03234201431274414,
-                0.032118785858154295,
-                0.03211673736572265,
-                0.031765504837036135,
-                0.0319866886138916,
-                0.03199692726135254,
-                0.032059391021728514,
-                0.031221759796142577,
-                0.03174297523498535,
-                0.02983628845214844,
-                0.031410175323486327,
-                0.03156991958618164,
-                0.03136614418029785,
-                0.029371391296386717,
-                0.029903104782104492,
-                0.028296960830688476,
-                0.028291072845458985,
-                0.028262399673461915,
-                0.028241920471191406,
-                0.028240896224975585,
-                0.02835456085205078,
-                0.028322816848754883,
-                0.028276735305786133,
-                0.028301311492919923,
                 0.028325887680053712,
-                0.02839347267150879,
                 0.02836992073059082,
-                0.028276735305786133,
-                0.028267520904541016,
-                0.02838937568664551,
-                0.028480512619018555,
-                0.02832486343383789,
-                0.028279008865356444,
-                0.02837481689453125,
-                0.028370943069458008,
-                0.02835456085205078,
-                0.02836582374572754,
-                0.02833510398864746,
                 0.028313600540161132,
-                0.028338176727294922,
-                0.02830745506286621,
                 0.028270591735839845,
-                0.028216320037841795,
-                0.028236799240112305,
-                0.028305408477783203,
-                0.028454912185668944,
-                0.028297216415405273,
-                0.028229631423950196,
-                0.028177408218383788,
-                0.028520448684692383,
                 0.028283903121948242,
-                0.028456960678100586,
-                0.02815180778503418,
-                0.028234752655029297,
-                0.02810163116455078,
                 0.028314624786376953,
-                0.02857062339782715,
-                0.028265472412109374,
-                0.028256256103515624,
-                0.02836479949951172,
-                0.02835353660583496,
-                0.02836992073059082,
                 0.028278783798217775,
-                0.02835251235961914,
-                0.0281395206451416,
-                0.028673023223876954,
-                0.028110847473144532,
-                0.02831667137145996,
-                0.028214271545410157,
-                0.028331008911132813,
-                0.028264448165893553,
                 0.02840985679626465,
-                0.028246015548706056,
-                0.02872319984436035,
-                0.028240896224975585,
-                0.028302335739135744,
-                0.028257280349731444,
-                0.028492799758911135,
-                0.028256320953369142,
-                0.028370880126953126,
-                0.028271615982055662,
-                0.028445695877075194,
-                0.028283903121948242,
-                0.028247039794921876,
-                0.02830335998535156,
-                0.028494848251342773,
                 0.02838425636291504,
-                0.028338176727294922,
-                0.028227807998657227,
-                0.028345119476318358,
-                0.028215295791625978,
-                0.028278783798217775,
-                0.02814668846130371,
                 0.028266496658325195,
-                0.028161024093627928,
-                0.028165119171142578,
-                0.02818252754211426,
-                0.028205055236816406,
-                0.027983871459960938,
-                0.02815180778503418,
-                0.028107776641845703,
                 0.028302335739135744,
-                0.02817638397216797
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 35.178374330310795
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1009.913856,
             "max_vram": 2585.788416,
             "max_reserved": 2101.346304,
             "max_allocated": 2014.912
         },
         "latency": {
             "unit": "s",
+            "mean": 0.03703229379126066,
+            "stdev": 0.0005140512718572568,
             "values": [
+                0.041216480255126954,
+                0.03949055862426758,
+                0.038371326446533204,
+                0.03802521514892578,
+                0.037797889709472655,
+                0.03784089660644531,
+                0.04021657562255859,
+                0.04187136077880859,
+                0.038798336029052735,
+                0.037601280212402347,
+                0.03723583984375,
+                0.03724627304077149,
+                0.03697663879394531,
+                0.03689574432373047,
+                0.03692031860351563,
+                0.036977664947509765,
+                0.03687014389038086,
+                0.03688755035400391,
+                0.03679334259033203,
+                0.03679155349731445,
+                0.03726847839355469,
                 0.03683225631713867,
+                0.036825088500976565,
+                0.036843521118164066,
+                0.03674009704589844,
+                0.03679846572875976,
+                0.037228801727294925,
+                0.03674828720092774,
+                0.03709337615966797,
+                0.03716505432128906,
+                0.036773887634277344,
+                0.03713433456420898,
+                0.03712307357788086,
+                0.037580799102783204,
+                0.03706070327758789,
+                0.0371701774597168,
+                0.037101566314697264,
+                0.03711385726928711,
+                0.037160961151123044,
+                0.03709747314453125,
+                0.03717324829101563,
+                0.03698483276367188,
+                0.0367710075378418,
+                0.03686809539794922,
                 0.036918270111083985,
+                0.03681398391723633,
+                0.03687628936767578,
+                0.03683020782470703,
+                0.036959232330322264,
+                0.03681382369995117,
+                0.036908031463623044,
+                0.03686118316650391,
+                0.036891841888427736,
+                0.03693260955810547,
+                0.03685273742675781,
+                0.03679452896118164,
+                0.03683430480957031,
+                0.03686707305908203,
+                0.036831230163574216,
+                0.036945919036865234,
+                0.036811775207519534,
                 0.036953086853027346,
+                0.03687014389038086,
+                0.036910079956054685,
+                0.037101566314697264,
+                0.03745792007446289,
+                0.03725321578979492,
+                0.03730124664306641,
+                0.036999168395996096,
                 0.03678003311157227,
                 0.03679641723632812,
+                0.03693056106567383,
+                0.03686297607421875,
+                0.036847614288330076,
+                0.03693772888183594,
+                0.03724185562133789,
+                0.03679948806762695,
+                0.037147647857666014,
+                0.03702374267578125,
                 0.036915199279785156,
+                0.03711081695556641,
+                0.03719887924194336,
+                0.03720499038696289,
+                0.03711695861816406,
+                0.03706982421875,
+                0.03690086364746094,
+                0.03696537780761719,
+                0.03718265533447265,
+                0.03718963241577149,
+                0.03692544174194336,
+                0.0368702392578125,
                 0.03684659194946289,
+                0.03687014389038086,
+                0.03685580825805664,
+                0.03693670272827149,
                 0.037010433197021485,
                 0.036942848205566405,
                 0.03688140869140625,
+                0.0369244155883789,
+                0.036857856750488284,
+                0.03677798461914063,
+                0.03681792068481445,
+                0.036782081604003904,
+                0.03681075286865235,
+                0.03684044647216797,
+                0.036738048553466796,
+                0.03684454345703125,
+                0.03688345718383789,
+                0.03696230316162109,
+                0.037085182189941404,
+                0.03685887908935547,
+                0.037093505859375,
+                0.03690291213989258,
+                0.036915199279785156,
+                0.03684259033203125,
                 0.036890625,
+                0.03690291213989258,
                 0.03696640014648438,
+                0.037144577026367184,
+                0.037308414459228514,
+                0.037348350524902346,
+                0.037379070281982424,
+                0.03705036926269531,
+                0.03698688125610351,
+                0.03698688125610351,
+                0.03722649765014648,
+                0.0372674560546875,
+                0.03702067184448242,
+                0.037010433197021485,
+                0.03747020721435547,
+                0.03693260955810547,
+                0.03698175811767578,
+                0.0370063362121582,
+                0.03688755035400391,
+                0.03696464157104492,
+                0.036926464080810545,
                 0.03693670272827149,
+                0.03685683059692383,
+                0.03699513626098633,
+                0.03699302291870117,
+                0.036918270111083985,
+                0.036985855102539066,
+                0.037375999450683595,
+                0.036898815155029296,
+                0.036927646636962894,
+                0.036913150787353514,
+                0.036982784271240236,
+                0.036920543670654296,
+                0.03700838470458984,
+                0.036988929748535154,
                 0.03687628936767578,
+                0.03690291213989258,
+                0.03681689453125,
+                0.03689686584472656,
+                0.03679846572875976,
                 0.036880382537841795,
+                0.03692134475708008,
+                0.03697356796264648,
+                0.036905086517333985,
+                0.03706572723388672,
+                0.03681894302368164,
+                0.036921600341796874,
+                0.03681280136108398,
+                0.03688652801513672,
+                0.03684659194946289,
+                0.03687833786010742,
+                0.0368353271484375,
+                0.036887870788574216,
+                0.036863998413085936,
                 0.036945919036865234,
+                0.03673190307617188,
+                0.03686912155151367,
                 0.03685990524291992,
+                0.0370247688293457,
                 0.03696537780761719,
                 0.03691622543334961,
+                0.036877311706542966,
+                0.03682304000854492,
+                0.03684966278076172,
+                0.036883487701416015,
+                0.03692134475708008,
+                0.0368353271484375,
+                0.036880382537841795,
+                0.03688447952270508,
+                0.037407745361328126,
+                0.03691622543334961,
+                0.03685171127319336,
+                0.03703091049194336,
+                0.03697459030151367,
+                0.03688550567626953,
+                0.03682304000854492,
+                0.03688550567626953,
+                0.036787200927734375,
+                0.03684454345703125,
+                0.03678412628173828,
+                0.03676803207397461,
+                0.03691417694091797,
+                0.03723775863647461,
+                0.036841472625732424,
+                0.03687116622924805,
+                0.036934654235839845,
+                0.036880382537841795,
+                0.0369541130065918,
+                0.03690188980102539,
                 0.036915199279785156,
+                0.03685990524291992,
+                0.03683327865600586,
+                0.03685887908935547,
+                0.037002239227294925,
+                0.03694182586669922,
+                0.036962558746337894,
+                0.03683225631713867,
+                0.036918270111083985,
+                0.03703398513793945,
+                0.03692246246337891,
+                0.036959232330322264,
+                0.03681587219238281,
+                0.036945919036865234,
                 0.03688243103027344,
+                0.036959232330322264,
+                0.03692544174194336,
+                0.03696230316162109,
+                0.036853759765625,
+                0.03692031860351563,
+                0.036907009124755856,
+                0.0370247688293457,
+                0.036860927581787106,
+                0.036969470977783206,
+                0.03687628936767578,
+                0.036915199279785156,
+                0.036754688262939456,
+                0.036893695831298826,
+                0.03686604690551758,
+                0.03684454345703125,
+                0.0369172477722168,
+                0.03696332931518555,
+                0.03690086364746094,
+                0.03701657485961914,
+                0.037238784790039066,
+                0.03691622543334961,
+                0.03681382369995117,
+                0.036994049072265625,
+                0.03682918548583984,
+                0.03694079971313476,
+                0.03696028900146484,
+                0.03695833587646485,
+                0.036923393249511716,
+                0.036988929748535154,
+                0.036972545623779295,
+                0.03706572723388672,
+                0.036931583404541016,
+                0.03733708953857422,
+                0.036874240875244144,
+                0.0368721923828125,
+                0.036916385650634764,
+                0.03696844863891602,
+                0.0369183349609375,
+                0.036946048736572264,
+                0.03693363189697266,
+                0.03698284912109375,
                 0.036947967529296875,
+                0.03700735855102539,
+                0.036992000579833983,
+                0.03701145553588867,
+                0.03693875122070313,
+                0.036953086853027346,
+                0.03674041748046875,
+                0.0369736328125,
+                0.03684048080444336,
+                0.03690822219848633,
+                0.03685273742675781
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 6912.88531688021
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 1009.913856,
             "max_vram": 2858.418176,
             "max_reserved": 2373.976064,
             "max_allocated": 2326.085632
         },
         "latency": {
             "unit": "s",
+            "mean": 7.244024317741396,
+            "stdev": 0.004989447593689356,
             "values": [
+                7.249013765335086,
+                7.239034870147707
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 35.20142793771102
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.02840793850094664,
+            "stdev": 0.00019315407561114144,
             "values": [
+                0.029053951263427736,
+                0.028392448425292968,
+                0.028463104248046874,
+                0.028463104248046874,
+                0.02838528060913086,
+                0.0283986873626709,
+                0.028428192138671874,
+                0.028367008209228516,
+                0.028359519958496095,
+                0.028430335998535155,
+                0.02840166473388672,
+                0.028322816848754883,
+                0.02834329605102539,
+                0.02832691192626953,
+                0.028408832550048828,
+                0.028498943328857423,
+                0.028465152740478516,
+                0.028429311752319338,
+                0.028297216415405273,
+                0.02834329605102539,
+                0.02837606430053711,
+                0.028421119689941408,
+                0.028333248138427733,
+                0.02850592041015625,
+                0.028374143600463867,
+                0.02839347267150879,
+                0.02842201614379883,
+                0.02875494384765625,
+                0.028363807678222656,
+                0.02842313575744629,
                 0.02835763168334961,
+                0.028366847991943358,
+                0.02837731170654297,
+                0.028475168228149415,
+                0.028412960052490235,
+                0.028468191146850588,
+                0.02831257629394531,
+                0.02841804885864258,
+                0.02835660743713379,
+                0.028419071197509766,
+                0.028685440063476564,
+                0.028428159713745117,
                 0.02835353660583496,
+                0.028428287506103517,
+                0.028785663604736327,
+                0.028375040054321288,
+                0.02834943962097168,
+                0.028439552307128906,
+                0.028455936431884765,
+                0.02840575981140137,
+                0.028338176727294922,
+                0.02877542304992676,
+                0.028358655929565428,
+                0.028476415634155275,
+                0.028492799758911135,
+                0.028387327194213868,
+                0.028468223571777345,
+                0.02840166473388672,
+                0.028440832138061523,
+                0.028384031295776366,
+                0.028553184509277345,
+                0.028410879135131836,
+                0.03211775970458984,
+                0.028556480407714843,
+                0.028443456649780274,
+                0.028383232116699218,
+                0.02838630485534668,
+                0.02855936050415039,
+                0.028443647384643556,
+                0.028481664657592772,
+                0.02838003158569336,
+                0.02852454376220703,
+                0.028449792861938477,
+                0.02834432029724121,
+                0.02837708854675293,
+                0.028356672286987305,
+                0.02842412757873535,
+                0.02843974494934082,
+                0.028511039733886717,
+                0.028443647384643556,
+                0.028400640487670898,
+                0.028478464126586913,
+                0.02834943962097168,
+                0.028367071151733397,
+                0.028371776580810547,
+                0.02843440055847168,
+                0.02846313667297363,
+                0.028473312377929688,
+                0.028483583450317384,
                 0.0283371524810791,
+                0.028400640487670898,
+                0.02849488067626953,
+                0.0284671688079834,
+                0.02836582374572754,
+                0.028358655929565428,
+                0.02835660743713379,
+                0.02836275291442871,
+                0.028474367141723633,
+                0.028836000442504884,
+                0.028378976821899413,
+                0.0283504638671875,
+                0.028473344802856446,
+                0.028402687072753906,
+                0.028473344802856446,
+                0.028468223571777345,
+                0.028496896743774414,
+                0.028423168182373046,
+                0.02843145561218262,
+                0.028446624755859375,
+                0.028390399932861327,
+                0.028431360244750976,
+                0.028417024612426758,
+                0.028308479309082032,
+                0.028528640747070313,
+                0.028342271804809572,
+                0.028818559646606446,
+                0.02842838478088379,
+                0.028426015853881836,
+                0.028427263259887696,
+                0.028458112716674804,
+                0.028477312088012696,
+                0.028442623138427735,
+                0.02839449691772461,
+                0.028499967575073244,
+                0.028404735565185548,
+                0.028482559204101563,
+                0.028407808303833007,
+                0.028410879135131836,
+                0.028484607696533205,
+                0.028481536865234375,
+                0.028461055755615236,
+                0.028444671630859376,
+                0.02836582374572754,
+                0.028404735565185548,
+                0.02852249526977539,
+                0.028507135391235353,
+                0.028419071197509766,
+                0.028472320556640625,
+                0.028443647384643556,
+                0.028404895782470702,
+                0.02864726448059082,
+                0.028556287765502928,
+                0.028437503814697264,
+                0.028462080001831053,
+                0.028370943069458008,
+                0.02834342384338379,
+                0.028390304565429687,
+                0.02841804885864258,
+                0.0285644474029541,
+                0.02837196731567383,
+                0.02834534454345703,
+                0.028468223571777345,
+                0.028430400848388673,
+                0.02846201515197754,
+                0.028539903640747072,
+                0.028413951873779295,
+                0.02835353660583496,
+                0.028482559204101563,
+                0.02832076835632324,
+                0.02838444709777832,
+                0.02827552032470703,
                 0.02834124755859375,
+                0.0283637752532959,
+                0.028454912185668944,
+                0.028437503814697264,
+                0.028407808303833007,
+                0.02838118362426758,
+                0.02838937568664551,
                 0.028309503555297853,
+                0.028862464904785157,
+                0.02851139259338379,
+                0.028392288208007814,
+                0.028421119689941408,
+                0.028422143936157225,
+                0.028391424179077147,
+                0.02835148811340332,
+                0.02835660743713379,
+                0.028377344131469726,
+                0.02839116859436035,
+                0.028471296310424804,
+                0.02836070442199707,
+                0.02837606430053711,
+                0.028399616241455077,
+                0.02834534454345703,
+                0.028303455352783204,
+                0.028364704132080077,
+                0.028820480346679687,
+                0.02838118362426758,
+                0.028358848571777343,
+                0.028378944396972656,
+                0.02829516792297363,
+                0.028408832550048828,
+                0.028251327514648438,
+                0.028241727828979494,
+                0.02841200065612793,
+                0.028364704132080077,
+                0.02830745506286621,
                 0.028416000366210937,
+                0.02832486343383789,
+                0.028497919082641602,
+                0.028309503555297853,
+                0.028399616241455077,
+                0.02838937568664551,
+                0.028403711318969727,
+                0.028446720123291015,
+                0.028404735565185548,
                 0.02830335998535156,
+                0.02840166473388672,
+                0.028344352722167968,
+                0.02835964775085449,
+                0.02832896041870117,
+                0.02836992073059082,
+                0.02834739112854004,
+                0.028321792602539062,
+                0.028239871978759764,
+                0.028261375427246094,
+                0.028269567489624024,
+                0.02829516792297363,
+                0.028292095184326172,
+                0.028280832290649413,
+                0.028257280349731444,
+                0.028394752502441407,
+                0.028260095596313477,
+                0.028417184829711915,
+                0.028330848693847655,
+                0.0283156795501709,
+                0.028314592361450196,
+                0.02836787223815918,
                 0.028286975860595705,
+                0.02828607940673828,
+                0.028283775329589842,
                 0.02862387275695801,
+                0.028473344802856446,
+                0.02835968017578125,
+                0.02839347267150879,
+                0.028368896484375,
+                0.02835660743713379,
+                0.028329984664916992,
+                0.028262399673461915,
+                0.028370943069458008,
+                0.028691455841064452,
+                0.02836070442199707,
+                0.028375040054321288,
+                0.028420095443725587,
+                0.02837708854675293,
+                0.02835660743713379,
                 0.028233728408813476,
+                0.028281024932861328,
+                0.02833286476135254,
+                0.028402687072753906,
                 0.028263423919677736,
+                0.028331008911132813,
+                0.0283504638671875,
+                0.02829516792297363,
+                0.028177408218383788,
+                0.028719104766845704,
+                0.02847551918029785,
+                0.02838924789428711,
+                0.028374176025390625,
+                0.028345184326171877,
+                0.02850201606750488,
+                0.028445695877075194,
+                0.02839449691772461,
+                0.028371103286743166,
+                0.028382047653198243,
+                0.028526655197143556,
+                0.028366783142089843,
+                0.028421119689941408,
+                0.0283637752532959,
+                0.02838528060913086,
+                0.028375263214111327,
+                0.028448543548583984,
+                0.02839449691772461,
+                0.02856038475036621,
+                0.028449792861938477,
+                0.028479488372802734,
                 0.02834534454345703,
+                0.02831987190246582,
+                0.028410751342773436,
+                0.028457984924316407,
+                0.028470272064208983,
+                0.02842748832702637,
+                0.028431135177612303,
+                0.02835558319091797,
+                0.02835148811340332,
+                0.028292095184326172,
+                0.028395519256591797,
+                0.028476415634155275,
+                0.028383232116699218,
                 0.028387327194213868,
+                0.02842233657836914,
+                0.028399423599243166,
+                0.028383232116699218,
+                0.028408832550048828,
+                0.028411903381347657,
+                0.028411903381347657,
                 0.02836787223815918,
+                0.02838835144042969,
+                0.02838528060913086,
                 0.028413951873779295,
+                0.028474367141723633,
+                0.028511295318603514,
+                0.028382335662841797,
+                0.028393280029296874,
+                0.028373088836669922,
+                0.028394399642944337,
+                0.028284927368164063,
+                0.028430335998535155,
+                0.028423168182373046,
+                0.028652544021606444,
+                0.028681215286254884,
+                0.028465152740478516,
                 0.028387327194213868,
+                0.028425216674804688,
+                0.028387327194213868,
+                0.028441600799560547,
+                0.02858598327636719,
+                0.028498943328857423,
                 0.028461055755615236,
+                0.028472320556640625,
+                0.028481536865234375,
+                0.02856345558166504,
+                0.028428287506103517,
+                0.028395519256591797,
+                0.028425216674804688,
+                0.028453887939453124,
+                0.02879078483581543,
+                0.02836582374572754,
+                0.028353759765625,
+                0.02839241600036621,
+                0.028436288833618165,
+                0.028276735305786133,
                 0.02840575981140137,
+                0.028400640487670898,
+                0.02849398422241211,
+                0.028372831344604492,
+                0.028413951873779295,
                 0.028440576553344726,
                 0.02837606430053711,
+                0.028382207870483397,
+                0.028390399932861327,
+                0.028382240295410158,
+                0.02836172866821289,
+                0.028453855514526366,
+                0.028475391387939454,
+                0.028404735565185548,
+                0.028391424179077147,
+                0.02834329605102539,
+                0.028427263259887696,
+                0.028416000366210937,
+                0.0283536319732666,
+                0.02832579231262207,
+                0.02836992073059082,
+                0.028366975784301758,
+                0.028426111221313475,
+                0.028368896484375,
+                0.028292095184326172,
                 0.02832793617248535,
+                0.028473344802856446,
+                0.02836479949951172,
                 0.02839756774902344,
+                0.028379167556762695,
+                0.028479455947875976,
                 0.028305408477783203,
+                0.02836479949951172,
+                0.02831667137145996,
                 0.028325887680053712,
+                0.028296192169189452,
+                0.02837196731567383,
+                0.028294143676757814,
+                0.02833612823486328,
+                0.028234752655029297,
                 0.02836992073059082,
+                0.028209152221679686,
+                0.0283371524810791,
+                0.02822675132751465,
+                0.028374847412109376,
+                0.02832076835632324,
+                0.02837606430053711,
+                0.028416000366210937,
+                0.028641279220581056,
+                0.028425216674804688,
+                0.028416000366210937,
+                0.028472320556640625,
+                0.028457984924316407,
+                0.028430335998535155,
+                0.028391424179077147,
+                0.028375040054321288,
+                0.028279903411865235,
+                0.028236831665039062,
+                0.02830732727050781,
+                0.02831667137145996,
+                0.028330175399780274,
+                0.0283123836517334,
+                0.028429311752319338,
+                0.028556287765502928,
+                0.02850624084472656,
+                0.028791807174682618,
+                0.02832579231262207,
+                0.02838217544555664,
+                0.028464128494262695,
+                0.028398591995239256,
+                0.028396543502807618,
+                0.02834841537475586,
+                0.02838425636291504,
+                0.02837708854675293,
+                0.028447872161865236,
+                0.028418943405151366,
+                0.028431360244750976,
+                0.02842643165588379,
+                0.028483392715454102,
+                0.02838630485534668,
                 0.028313600540161132,
+                0.028277759552001954,
+                0.02854400062561035,
                 0.028270591735839845,
+                0.028271615982055662,
+                0.028357824325561522,
+                0.02831955146789551,
+                0.028306655883789063,
+                0.028366624832153322,
                 0.028283903121948242,
+                0.028298431396484375,
+                0.028388160705566406,
+                0.028314687728881835,
+                0.028319679260253906,
                 0.028314624786376953,
+                0.02834841537475586,
                 0.028278783798217775,
+                0.02831974411010742,
+                0.028209184646606444,
+                0.028265439987182617,
+                0.02827270317077637,
+                0.028287935256958007,
+                0.028434431076049805,
+                0.028313695907592775,
+                0.0283288631439209,
+                0.02833612823486328,
                 0.02840985679626465,
+                0.028386367797851562,
+                0.028263359069824218,
+                0.02840073585510254,
+                0.028315711975097656,
+                0.028368736267089845,
+                0.02834022331237793,
                 0.02838425636291504,
+                0.02838015937805176,
+                0.028420095443725587,
+                0.02834124755859375,
+                0.028298240661621094,
+                0.02838243293762207,
+                0.02853660774230957,
+                0.02844281578063965,
+                0.02834127998352051,
+                0.028352287292480467,
+                0.028203168869018556,
+                0.0283851203918457,
+                0.028429471969604492,
+                0.028372831344604492,
+                0.028271615982055662,
+                0.02862387275695801,
+                0.028215423583984375,
+                0.028498815536499023,
+                0.028403711318969727,
+                0.02840575981140137,
+                0.02833510398864746,
+                0.02834124755859375,
+                0.028305408477783203,
+                0.028829696655273438,
+                0.02830745506286621,
+                0.028261375427246094,
+                0.028290048599243164,
+                0.028422239303588868,
+                0.028380064010620116,
+                0.028422143936157225,
+                0.02832896041870117,
+                0.028313600540161132,
+                0.02836479949951172,
+                0.0283637752532959,
+                0.028235776901245117,
+                0.02834329605102539,
+                0.028300287246704102,
+                0.028451839447021485,
+                0.02831564712524414,
+                0.028484607696533205,
+                0.028422143936157225,
+                0.02834943962097168,
+                0.028281856536865234,
                 0.028266496658325195,
+                0.028297216415405273,
+                0.028455968856811523,
+                0.028338144302368164,
+                0.028422208786010743,
+                0.028363712310791017,
+                0.028345407485961913,
+                0.028309440612792967,
+                0.028444671630859376,
+                0.028345439910888674,
+                0.02834223937988281,
+                0.02831660842895508,
+                0.02839347267150879,
+                0.02839449691772461,
+                0.028424192428588867,
+                0.0283371524810791,
+                0.02833510398864746,
                 0.028302335739135744,
+                0.028375040054321288,
+                0.028282880783081055,
+                0.028521472930908204,
+                0.028213247299194336
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 35.20142793771104
         },
         "energy": null,
         "efficiency": null