kreas
/

MA_benchmarks

Model card Files Files and versions Community

kreas commited on Apr 19

Commit

7db6534

•

1 Parent(s): 38d11f6

Upload /Mistral-7B-v0.1-GPTQ-4bit/gptq4bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Browse files

Files changed (1) hide show

Mistral-7B-v0.1-GPTQ-4bit/gptq4bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json +694 -694

Mistral-7B-v0.1-GPTQ-4bit/gptq4bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json CHANGED Viewed

@@ -2,257 +2,257 @@
     "prefill": {
         "memory": {
             "unit": "MB",
-            "max_ram": 7501.16864,
             "max_vram": 5544.869888,
             "max_reserved": 5058.330624,
             "max_allocated": 4906.976256
         },
         "latency": {
             "unit": "s",
-            "mean": 0.04247161491846634,
-            "stdev": 9.919363500733545e-05,
             "values": [
-                0.04393843078613281,
-                0.04244582366943359,
-                0.042456062316894534,
-                0.042439678192138675,
-                0.042433536529541016,
-                0.04244377517700195,
-                0.04245401763916016,
-                0.042466304779052735,
-                0.042447872161865234,
-                0.04246441650390625,
-                0.042651649475097655,
-                0.04245913696289062,
-                0.042446849822998046,
-                0.04242038345336914,
-                0.04247859191894531,
-                0.04244889450073242,
-                0.04245094299316406,
-                0.04244070434570312,
-                0.04247552108764648,
-                0.042433536529541016,
-                0.04245503997802735,
-                0.042472446441650394,
-                0.04245094299316406,
-                0.04247552108764648,
-                0.04244582366943359,
-                0.04245401763916016,
-                0.04245913696289062,
-                0.04244172668457031,
-                0.04243558502197266,
-                0.0424356803894043,
-                0.04244582366943359,
-                0.042452991485595705,
                 0.04245401763916016,
-                0.04245708847045898,
-                0.042444801330566405,
                 0.042464256286621094,
-                0.04245811080932617,
-                0.04245913696289062,
-                0.042485759735107424,
-                0.042485759735107424,
-                0.04248883056640625,
-                0.042449920654296876,
-                0.042449920654296876,
-                0.04246220779418945,
-                0.04246835327148438,
-                0.042461185455322265,
-                0.042469375610351565,
                 0.04244889450073242,
                 0.04245913696289062,
                 0.042461185455322265,
-                0.04248883056640625,
-                0.042466304779052735,
-                0.04243558502197266,
-                0.042482688903808595,
-                0.04247964859008789,
                 0.042449920654296876,
-                0.0424736328125,
-                0.042444801330566405,
-                0.04247552108764648,
-                0.042472446441650394,
-                0.042472446441650394,
                 0.04244275283813476,
-                0.0424376335144043,
-                0.04242124938964844,
-                0.04246527862548828,
                 0.042472446441650394,
-                0.0424683837890625,
-                0.04248166275024414,
-                0.042444801330566405,
-                0.04240793609619141,
-                0.042464256286621094,
                 0.042447872161865234,
-                0.042464256286621094,
-                0.042464256286621094,
-                0.042477569580078124,
-                0.04245913696289062,
                 0.042449920654296876,
-                0.042439678192138675,
-                0.042501121520996096,
-                0.042447872161865234,
-                0.04247071838378906,
                 0.04245708847045898,
-                0.04246239852905274,
-                0.042493953704833984,
                 0.04245401763916016,
-                0.04248473739624024,
-                0.04247040176391602,
-                0.04248166275024414,
-                0.042461185455322265,
-                0.042446849822998046,
                 0.04247552108764648,
-                0.042469375610351565,
-                0.04246527862548828,
                 0.04246527862548828,
-                0.04245811080932617,
-                0.04248166275024414,
-                0.04246227264404297,
-                0.04244889450073242,
-                0.042461185455322265,
-                0.04244889450073242,
-                0.0424796142578125,
                 0.04244889450073242,
                 0.042472446441650394,
                 0.04245401763916016,
                 0.042464256286621094,
-                0.04246527862548828,
-                0.04245708847045898,
-                0.04244889450073242,
-                0.04248064041137695,
-                0.04245119857788086,
                 0.04245196914672852,
-                0.04244275283813476,
-                0.042464256286621094,
-                0.042439678192138675,
                 0.042466304779052735,
-                0.04245094299316406,
-                0.04248064041137695,
-                0.04244275283813476,
-                0.042482688903808595,
-                0.042452991485595705,
-                0.0424796142578125,
                 0.042461185455322265,
-                0.042427455902099606,
-                0.04244582366943359,
                 0.04248064041137695,
                 0.04246015930175781,
-                0.04244889450073242,
-                0.042472446441650394,
-                0.04246732711791992,
                 0.04244582366943359,
-                0.04248883056640625,
-                0.042482688903808595,
-                0.04248883056640625,
                 0.04248064041137695,
-                0.042472446441650394,
-                0.042447872161865234,
-                0.042501121520996096,
-                0.04244889450073242,
-                0.042444801330566405,
-                0.04247449493408203,
-                0.042426368713378904,
-                0.04248678588867188,
-                0.042493247985839845,
-                0.04246220779418945,
-                0.0424376335144043,
-                0.04247347259521484,
-                0.0424898567199707,
-                0.04244095993041992,
                 0.04245708847045898,
-                0.04246527862548828,
-                0.042463294982910155,
                 0.042472446441650394,
-                0.04251238250732422,
-                0.04245094299316406,
-                0.04244889450073242,
-                0.04247449493408203,
-                0.042477790832519534,
-                0.04248064041137695,
-                0.042461185455322265,
                 0.04248166275024414,
                 0.04246527862548828,
-                0.042449920654296876,
-                0.04246220779418945,
-                0.042449920654296876,
-                0.042471424102783206,
-                0.04246428680419922,
-                0.0424898567199707,
-                0.0424376335144043,
-                0.04246835327148438,
-                0.042466304779052735,
-                0.042472446441650394,
-                0.042469375610351565,
-                0.042471424102783206,
-                0.04244582366943359,
-                0.04248473739624024,
-                0.04249711990356445,
-                0.04249190521240234,
-                0.04246656036376953,
-                0.0424796142578125,
                 0.04245811080932617,
-                0.04245935821533203,
-                0.04248681640625,
-                0.04247766494750976,
                 0.04247552108764648,
-                0.04249497604370117,
-                0.042439678192138675,
-                0.042459297180175784,
-                0.042469375610351565,
-                0.04248883056640625,
-                0.042444801330566405,
-                0.042501121520996096,
-                0.042447872161865234,
                 0.0424796142578125,
-                0.042487808227539066,
                 0.04247552108764648,
-                0.042448001861572264,
-                0.04247040176391602,
-                0.04244377517700195,
-                0.04248166275024414,
-                0.04247859191894531,
-                0.04246732711791992,
-                0.042472446441650394,
-                0.04246432113647461,
-                0.042708992004394535,
-                0.04246025466918945,
-                0.04246220779418945,
                 0.04245708847045898,
-                0.042447872161865234,
-                0.04247654342651367,
-                0.04247654342651367,
-                0.042469375610351565,
-                0.04246531295776367,
-                0.042452991485595705,
                 0.04245811080932617,
-                0.042477790832519534,
-                0.042471424102783206,
                 0.04247040176391602,
-                0.04244073486328125,
-                0.0424901123046875,
                 0.042464256286621094,
-                0.042472446441650394,
-                0.042460224151611325,
-                0.04246732711791992,
-                0.042471424102783206,
                 0.042482688903808595,
                 0.042446849822998046,
                 0.04246323013305664,
-                0.04248089599609375,
-                0.042491935729980466,
-                0.04247654342651367,
                 0.04248064041137695,
-                0.042449920654296876,
                 0.04247552108764648,
-                0.042461185455322265,
-                0.04248473739624024,
-                0.042466304779052735
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 6027.555121024915
         },
         "energy": null,
         "efficiency": null
@@ -260,23 +260,23 @@
     "decode": {
         "memory": {
             "unit": "MB",
-            "max_ram": 7503.265792,
             "max_vram": 5716.836352,
             "max_reserved": 5230.297088,
             "max_allocated": 5031.01696
         },
         "latency": {
             "unit": "s",
-            "mean": 5.773392906188963,
-            "stdev": 0.0043458652496339845,
             "values": [
-                5.769047040939329,
-                5.777738771438597
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 44.1681354696378
         },
         "energy": null,
         "efficiency": null
@@ -285,524 +285,524 @@
         "memory": null,
         "latency": {
             "unit": "s",
-            "mean": 0.022640756494858666,
-            "stdev": 0.0005619443258706103,
             "values": [
-                0.025393152236938478,
-                0.023914560317993164,
-                0.02256377601623535,
-                0.022544384002685547,
-                0.022500352859497072,
-                0.022512800216674806,
-                0.022468448638916016,
-                0.022534143447875975,
-                0.02270524787902832,
-                0.022516639709472656,
-                0.02248806381225586,
-                0.022373376846313478,
-                0.0224399356842041,
-                0.022460416793823244,
-                0.022466560363769532,
-                0.022647808074951172,
-                0.022384639739990234,
-                0.022408191680908202,
-                0.022310047149658202,
-                0.022412128448486328,
-                0.022372352600097657,
-                0.02246143913269043,
-                0.02244710350036621,
-                0.02255564880371094,
-                0.02235215950012207,
-                0.02240278434753418,
-                0.02246348762512207,
-                0.022421503067016603,
-                0.022373376846313478,
-                0.022392831802368163,
-                0.02243071937561035,
-                0.02248908805847168,
-                0.022427648544311524,
-                0.022399200439453124,
-                0.022379295349121094,
-                0.022364288330078124,
-                0.02233843231201172,
-                0.02246963119506836,
-                0.02233679962158203,
-                0.0225248966217041,
-                0.022320192337036134,
-                0.022371007919311525,
-                0.022312959671020507,
-                0.02227609634399414,
-                0.022322175979614257,
-                0.022364160537719727,
-                0.022399999618530272,
-                0.022537216186523438,
-                0.0224716796875,
-                0.022427648544311524,
-                0.022426624298095704,
-                0.022383615493774413,
-                0.022445056915283205,
-                0.022436864852905275,
-                0.022378496170043945,
-                0.02246963119506836,
-                0.02241535949707031,
-                0.022377471923828125,
-                0.022387903213500978,
-                0.02239366340637207,
-                0.022403072357177735,
-                0.02230886459350586,
-                0.022336511611938475,
-                0.022577152252197266,
-                0.022334463119506837,
-                0.02237353515625,
-                0.022425535202026368,
-                0.02244803237915039,
-                0.02242355155944824,
-                0.0223191032409668,
-                0.022424575805664062,
-                0.02251263999938965,
-                0.022427648544311524,
-                0.022460416793823244,
-                0.02249932861328125,
-                0.022321151733398437,
-                0.022406143188476564,
-                0.02248297691345215,
-                0.022485984802246093,
-                0.022508544921875,
-                0.022465791702270508,
-                0.02247760009765625,
-                0.023648223876953124,
-                0.022518783569335937,
-                0.022335487365722655,
-                0.02327859115600586,
-                0.023742464065551756,
-                0.022545408248901368,
-                0.02242252731323242,
-                0.02237353515625,
-                0.02233123207092285,
-                0.022405120849609376,
-                0.022427648544311524,
-                0.022552576065063477,
-                0.02249932861328125,
-                0.022535167694091796,
-                0.022405248641967773,
-                0.022448223114013673,
-                0.022604639053344727,
-                0.022947776794433595,
-                0.022914047241210937,
-                0.022476959228515624,
-                0.02231996726989746,
-                0.022519807815551757,
-                0.022722560882568358,
-                0.02250979232788086,
-                0.022420255661010743,
-                0.02246348762512207,
-                0.02243174362182617,
-                0.022419519424438476,
-                0.022465471267700196,
-                0.022914047241210937,
-                0.022969343185424804,
-                0.022795263290405272,
-                0.022837247848510742,
-                0.022939647674560547,
-                0.022861824035644532,
-                0.022200319290161134,
-                0.022336544036865233,
-                0.022430688858032226,
-                0.022364160537719727,
-                0.022471935272216796,
-                0.022432512283325195,
-                0.022383615493774413,
-                0.022311935424804686,
-                0.022369279861450195,
-                0.022849536895751952,
-                0.022809600830078124,
-                0.03188534355163574,
-                0.029729759216308594,
-                0.02269696044921875,
-                0.02257935905456543,
-                0.022663135528564454,
-                0.02254732894897461,
-                0.022642688751220705,
-                0.022610944747924806,
-                0.022503488540649413,
-                0.022532032012939452,
-                0.02254745674133301,
-                0.022598655700683593,
-                0.022587423324584962,
-                0.02253615951538086,
-                0.02264575958251953,
-                0.02253209686279297,
-                0.022520832061767578,
-                0.022511615753173828,
-                0.02255564880371094,
-                0.022531232833862304,
-                0.022481760025024413,
-                0.02267033576965332,
-                0.022511615753173828,
-                0.022579200744628908,
-                0.02248192024230957,
-                0.02248089599609375,
-                0.0224849910736084,
-                0.022434816360473633,
-                0.022558719635009765,
-                0.022495231628417968,
-                0.022500352859497072,
-                0.022471935272216796,
-                0.02254003143310547,
-                0.02252288055419922,
-                0.02246246337890625,
-                0.022553600311279298,
-                0.022502592086791992,
-                0.022482751846313476,
-                0.022548479080200197,
-                0.022494207382202147,
-                0.02254745674133301,
-                0.022549535751342772,
-                0.02251772880554199,
-                0.022559743881225586,
-                0.023778400421142577,
-                0.022663072586059572,
-                0.022542335510253905,
-                0.022597631454467772,
-                0.02280243110656738,
-                0.02261299133300781,
-                0.022544384002685547,
-                0.022593536376953126,
-                0.0224768009185791,
-                0.022518783569335937,
-                0.02253926467895508,
-                0.022615039825439453,
-                0.022558975219726562,
-                0.022537984848022462,
-                0.022558719635009765,
-                0.02251692771911621,
-                0.022532928466796876,
-                0.022562047958374024,
-                0.022596351623535155,
-                0.022595584869384764,
-                0.022602752685546876,
-                0.02254745674133301,
-                0.022664192199707032,
-                0.022560768127441407,
-                0.022624256134033204,
-                0.022594560623168947,
-                0.022590463638305663,
-                0.022571008682250978,
-                0.02305433654785156,
-                0.02266726493835449,
-                0.022628448486328126,
-                0.02264463996887207,
-                0.022666240692138673,
-                0.022587392807006838,
-                0.022663232803344726,
-                0.022523839950561522,
-                0.0225218563079834,
-                0.022529247283935547,
-                0.022600479125976562,
-                0.022495231628417968,
-                0.022583295822143554,
-                0.022616064071655274,
-                0.022562816619873048,
-                0.022583295822143554,
-                0.02252720069885254,
-                0.022514463424682617,
-                0.022616064071655274,
-                0.022492160797119142,
-                0.02253824043273926,
-                0.022518783569335937,
-                0.022561792373657227,
-                0.022525951385498046,
-                0.022550752639770508,
-                0.022599456787109375,
-                0.02267136001586914,
-                0.022664192199707032,
-                0.022599679946899414,
-                0.022600704193115235,
-                0.02269696044921875,
                 0.02266009521484375,
-                0.023018495559692383,
-                0.02270310401916504,
-                0.02266316795349121,
-                0.022582271575927734,
                 0.022575103759765625,
-                0.02265088081359863,
-                0.02270515251159668,
-                0.022551551818847656,
-                0.022567935943603516,
-                0.022552576065063477,
-                0.022541311264038084,
-                0.022558719635009765,
-                0.022614015579223632,
-                0.022587392807006838,
-                0.022633472442626954,
-                0.0226375675201416,
-                0.02254643249511719,
-                0.022897663116455077,
-                0.022585344314575196,
-                0.02251263999938965,
                 0.022556671142578123,
-                0.0224901123046875,
-                0.022408191680908202,
-                0.022818912506103517,
-                0.022800416946411134,
-                0.022662336349487305,
-                0.02265465545654297,
-                0.02262937545776367,
-                0.022647808074951172,
-                0.022597759246826173,
-                0.022836095809936525,
-                0.022799360275268556,
-                0.022557695388793944,
-                0.022605823516845702,
-                0.02264473533630371,
-                0.022656000137329102,
-                0.02267033576965332,
-                0.022593536376953126,
-                0.022784000396728517,
-                0.02265907287597656,
-                0.022683647155761717,
-                0.02260406494140625,
-                0.022675167083740233,
-                0.022595903396606446,
-                0.022673088073730467,
-                0.022616064071655274,
-                0.022777856826782225,
-                0.022565887451171874,
-                0.022816768646240236,
-                0.022656000137329102,
-                0.02265088081359863,
-                0.022558719635009765,
-                0.022718463897705078,
-                0.022618431091308594,
-                0.022737600326538085,
-                0.022616064071655274,
                 0.022603776931762694,
-                0.022608896255493165,
                 0.022603776931762694,
-                0.02256719970703125,
-                0.022680320739746095,
-                0.022625280380249024,
-                0.02273481559753418,
-                0.022632448196411133,
-                0.022577152252197266,
-                0.02263859176635742,
-                0.022658048629760744,
-                0.022754304885864256,
-                0.022567935943603516,
-                0.022626304626464845,
-                0.022601728439331056,
-                0.022654144287109376,
-                0.022558752059936522,
-                0.02252060890197754,
                 0.022569984436035157,
-                0.022647808074951172,
-                0.022623231887817383,
-                0.02264089584350586,
-                0.022690559387207033,
-                0.022552576065063477,
                 0.022615039825439453,
-                0.022619136810302733,
-                0.022635583877563478,
-                0.02264790344238281,
-                0.022656864166259765,
-                0.022693119049072265,
-                0.02276531219482422,
-                0.022623296737670898,
-                0.02265100860595703,
-                0.02261382484436035,
-                0.02267750358581543,
-                0.02267750358581543,
-                0.0227061767578125,
-                0.02270412826538086,
-                0.022784223556518556,
-                0.022608671188354492,
-                0.022694047927856446,
-                0.022616928100585937,
-                0.022642688751220705,
-                0.022642688751220705,
-                0.02255462455749512,
-                0.02266828727722168,
-                0.02275328063964844,
-                0.022666240692138673,
-                0.02262118339538574,
-                0.023946239471435548,
-                0.02260479927062988,
-                0.022614015579223632,
-                0.0226212158203125,
-                0.022736864089965822,
-                0.02283417510986328,
-                0.022722560882568358,
-                0.022732799530029296,
-                0.02264371109008789,
-                0.02265292739868164,
-                0.022609952926635743,
-                0.02267747116088867,
-                0.02269388771057129,
-                0.022838272094726563,
-                0.02284441566467285,
-                0.02268057632446289,
-                0.02270515251159668,
-                0.022574079513549804,
-                0.02267136001586914,
-                0.0226693115234375,
-                0.02267136001586914,
-                0.022787200927734376,
-                0.02270502471923828,
-                0.02266214370727539,
-                0.02271129608154297,
-                0.02269696044921875,
-                0.02269081687927246,
                 0.02266111946105957,
-                0.02268671989440918,
-                0.022767616271972657,
-                0.022610944747924806,
-                0.02263654327392578,
-                0.022664192199707032,
-                0.022750207901000977,
-                0.022607872009277344,
-                0.022552576065063477,
-                0.022627328872680662,
-                0.022614015579223632,
-                0.022623231887817383,
-                0.022584320068359375,
-                0.02265497589111328,
                 0.022510591506958007,
-                0.022591487884521484,
-                0.022641664505004884,
-                0.02259660720825195,
-                0.022556671142578123,
-                0.022624256134033204,
-                0.0225230712890625,
-                0.022604608535766603,
-                0.022649951934814453,
-                0.02264569664001465,
-                0.022674400329589842,
                 0.02255462455749512,
-                0.02253926467895508,
-                0.02253107261657715,
-                0.022597759246826173,
-                0.02249715232849121,
-                0.0225218563079834,
-                0.022651968002319337,
-                0.022516672134399413,
                 0.022579200744628908,
-                0.022562816619873048,
-                0.022615135192871092,
-                0.0225053768157959,
-                0.0225218563079834,
-                0.022611967086791994,
-                0.023559167861938478,
                 0.022623231887817383,
-                0.022735136032104492,
-                0.022677215576171875,
-                0.02264886474609375,
-                0.022617055892944337,
-                0.022772735595703125,
-                0.022594560623168947,
-                0.022645055770874025,
-                0.022566591262817383,
-                0.022674432754516603,
                 0.022579200744628908,
-                0.02266316795349121,
-                0.022558719635009765,
-                0.02259689521789551,
-                0.022527711868286133,
-                0.022640640258789063,
-                0.022593536376953126,
-                0.022578399658203126,
-                0.022630176544189452,
-                0.022656000137329102,
                 0.022527999877929687,
-                0.022635520935058592,
-                0.022615039825439453,
-                0.02309427261352539,
-                0.022607872009277344,
-                0.022707199096679686,
                 0.02271027183532715,
-                0.022641664505004884,
-                0.02272972869873047,
-                0.022563840866088865,
-                0.022649856567382814,
                 0.02267136001586914,
-                0.022595584869384764,
-                0.02264575958251953,
-                0.02269705581665039,
-                0.0228822078704834,
-                0.022579200744628908,
-                0.02264371109008789,
-                0.022567935943603516,
-                0.02266316795349121,
-                0.022576383590698242,
-                0.022651647567749022,
-                0.022649856567382814,
-                0.022725631713867187,
-                0.02261840057373047,
-                0.02258505630493164,
-                0.022735071182250977,
-                0.02269673538208008,
-                0.022722560882568358,
-                0.02269593620300293,
-                0.022635520935058592,
-                0.02308710479736328,
-                0.022704256057739257,
-                0.022632320404052733,
-                0.022691839218139647,
                 0.022692863464355468,
-                0.02269388771057129,
-                0.022635520935058592,
-                0.022658048629760744,
-                0.022665216445922853,
-                0.022658048629760744,
-                0.022617120742797852,
-                0.022613216400146484,
-                0.0225729923248291,
-                0.02261689567565918,
-                0.02270412826538086,
-                0.02263350486755371,
-                0.02258940887451172,
-                0.022654144287109376,
-                0.0225861759185791,
-                0.022657024383544923,
-                0.0227194881439209,
-                0.022608896255493165,
-                0.02249830436706543,
                 0.02253225517272949,
-                0.02266409683227539,
-                0.02256480026245117,
-                0.02261734390258789,
-                0.022640384674072266,
-                0.022627328872680662,
                 0.022614015579223632,
-                0.02271129608154297,
-                0.02269388771057129,
-                0.02308403205871582,
-                0.02263654327392578,
-                0.02265292739868164,
-                0.022671424865722656,
-                0.02266208076477051,
                 0.022631423950195313,
                 0.022608896255493165,
-                0.022644767761230467,
-                0.02257814407348633,
-                0.02254643249511719,
-                0.022716415405273437,
-                0.0227194881439209,
-                0.02269593620300293,
                 0.02265907287597656,
-                0.022682783126831054,
-                0.02254934310913086,
-                0.022541311264038084,
-                0.022573312759399413,
-                0.02260966491699219,
-                0.022487039566040038,
                 0.022598655700683593,
-                0.022850719451904297,
-                0.022495071411132814
             ]
         },
         "throughput": {
             "unit": "tokens/s",
-            "value": 44.168135469637825
         },
         "energy": null,
         "efficiency": null

     "prefill": {
         "memory": {
             "unit": "MB",
+            "max_ram": 7970.185216,
             "max_vram": 5544.869888,
             "max_reserved": 5058.330624,
             "max_allocated": 4906.976256
         },
         "latency": {
             "unit": "s",
+            "mean": 0.04247605333489883,
+            "stdev": 9.94106394560766e-05,
             "values": [
+                0.043952926635742184,
+                0.042403839111328126,
+                0.04242227172851563,
+                0.04244790267944336,
                 0.04245401763916016,
                 0.042464256286621094,
+                0.04247654342651367,
+                0.042422496795654296,
+                0.04246015930175781,
+                0.04246527862548828,
+                0.04246527862548828,
+                0.04244377517700195,
+                0.04246323013305664,
                 0.04244889450073242,
+                0.04245811080932617,
+                0.042487808227539066,
+                0.042464256286621094,
+                0.04243865585327149,
+                0.042482688903808595,
+                0.042471424102783206,
+                0.042446849822998046,
+                0.042447872161865234,
+                0.04247552108764648,
                 0.04245913696289062,
+                0.04246015930175781,
                 0.042461185455322265,
+                0.04242227172851563,
                 0.042449920654296876,
+                0.04245721435546875,
+                0.042450977325439454,
+                0.042464256286621094,
+                0.04247654342651367,
+                0.042485759735107424,
                 0.04244275283813476,
+                0.04244297790527344,
                 0.042472446441650394,
+                0.04248371124267578,
+                0.04247859191894531,
+                0.04243251037597656,
+                0.042436607360839845,
+                0.04243865585327149,
+                0.04249190521240234,
                 0.042447872161865234,
+                0.04245811080932617,
+                0.04246527862548828,
+                0.04246732711791992,
+                0.04247449493408203,
+                0.04247859191894531,
+                0.042456062316894534,
+                0.04246835327148438,
+                0.04248371124267578,
                 0.042449920654296876,
                 0.04245708847045898,
+                0.042456062316894534,
+                0.04247859191894531,
                 0.04245401763916016,
                 0.04247552108764648,
+                0.042449920654296876,
+                0.04246025466918945,
+                0.04245913696289062,
+                0.04248371124267578,
+                0.04245427322387695,
+                0.04245196914672852,
+                0.04245622253417969,
+                0.04247040176391602,
+                0.04243865585327149,
+                0.042472576141357424,
                 0.04246527862548828,
+                0.04247577667236328,
                 0.04244889450073242,
+                0.04247040176391602,
                 0.042472446441650394,
+                0.042505214691162106,
+                0.04245811080932617,
+                0.04247654342651367,
+                0.04247449493408203,
                 0.04245401763916016,
                 0.042464256286621094,
+                0.042466304779052735,
+                0.04245417785644531,
+                0.042471424102783206,
                 0.04245196914672852,
+                0.04246835327148438,
                 0.042466304779052735,
+                0.04246835327148438,
+                0.042431488037109374,
+                0.04247859191894531,
+                0.042472446441650394,
+                0.042464256286621094,
+                0.04245401763916016,
+                0.04242432022094727,
+                0.04246323013305664,
                 0.042461185455322265,
+                0.042510337829589843,
                 0.04248064041137695,
+                0.042456062316894534,
+                0.04250640106201172,
+                0.042449920654296876,
                 0.04246015930175781,
+                0.04245094299316406,
+                0.04245817565917969,
+                0.04244275283813476,
+                0.042487808227539066,
+                0.04245503997802735,
                 0.04244582366943359,
+                0.04244582366943359,
+                0.042485759735107424,
+                0.04242739105224609,
+                0.042485759735107424,
+                0.042490974426269534,
                 0.04248064041137695,
+                0.042471424102783206,
+                0.0424796142578125,
+                0.0424796142578125,
+                0.04245503997802735,
+                0.04246025466918945,
                 0.04245708847045898,
+                0.04247552108764648,
+                0.04246015930175781,
                 0.042472446441650394,
+                0.042469375610351565,
+                0.04246220779418945,
+                0.04244275283813476,
+                0.04245811080932617,
+                0.04269055938720703,
+                0.04245811080932617,
+                0.04247040176391602,
+                0.04246220779418945,
                 0.04248166275024414,
+                0.042452991485595705,
                 0.04246527862548828,
+                0.04245708847045898,
+                0.04245503997802735,
+                0.04250316619873047,
+                0.04248886489868164,
+                0.042446849822998046,
+                0.04245401763916016,
                 0.04245811080932617,
+                0.04248064041137695,
+                0.04250009536743164,
+                0.04247040176391602,
+                0.04249836730957031,
+                0.04247552108764648,
+                0.042445022583007815,
+                0.04247449493408203,
                 0.04247552108764648,
                 0.0424796142578125,
+                0.04244070434570312,
                 0.04247552108764648,
+                0.04245932769775391,
+                0.04247468948364258,
+                0.04247552108764648,
+                0.042466529846191405,
                 0.04245708847045898,
+                0.04248166275024414,
                 0.04245811080932617,
+                0.042485759735107424,
+                0.042452991485595705,
+                0.04247552108764648,
+                0.04248166275024414,
                 0.04247040176391602,
+                0.04246220779418945,
+                0.0424725456237793,
+                0.042447872161865234,
+                0.04246835327148438,
+                0.04244889450073242,
+                0.04248473739624024,
+                0.042466304779052735,
+                0.04247449493408203,
+                0.04245328140258789,
                 0.042464256286621094,
+                0.042447872161865234,
+                0.04250944137573242,
                 0.042482688903808595,
+                0.04247347259521484,
+                0.04248166275024414,
+                0.04248883056640625,
+                0.04244911956787109,
+                0.04248499298095703,
                 0.042446849822998046,
+                0.04247552108764648,
+                0.04244390487670899,
+                0.04248883056640625,
+                0.04245401763916016,
                 0.04246323013305664,
+                0.042482688903808595,
+                0.042485759735107424,
+                0.042456062316894534,
+                0.04247347259521484,
+                0.042466529846191405,
+                0.04244070434570312,
+                0.04249497604370117,
+                0.04247859191894531,
+                0.04246835327148438,
+                0.04246323013305664,
+                0.04244079971313477,
+                0.04248678588867188,
+                0.04245094299316406,
+                0.04245708847045898,
+                0.04246323013305664,
+                0.04249702453613281,
+                0.042482688903808595,
+                0.04249190521240234,
+                0.04247040176391602,
+                0.04250009536743164,
                 0.04248064041137695,
+                0.042510337829589843,
+                0.0425164794921875,
+                0.042493953704833984,
+                0.04248281478881836,
                 0.04247552108764648,
+                0.04247859191894531,
+                0.04248678588867188,
+                0.042499073028564455,
+                0.042499073028564455,
+                0.04249702453613281,
+                0.04252364730834961,
+                0.04251238250732422,
+                0.04249292755126953,
+                0.04247859191894531,
+                0.04249292755126953,
+                0.04248678588867188,
+                0.04248678588867188,
+                0.04249087905883789,
+                0.042502143859863284,
+                0.04246835327148438,
+                0.04252876663208008,
+                0.042510337829589843,
+                0.042501121520996096,
+                0.042474624633789065,
+                0.04252278518676758,
+                0.042464256286621094,
+                0.04248371124267578,
+                0.04247654342651367,
+                0.04250223922729492,
+                0.04247859191894531
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 6026.925288505261
         },
         "energy": null,
         "efficiency": null
     "decode": {
         "memory": {
             "unit": "MB",
+            "max_ram": 7972.282368,
             "max_vram": 5716.836352,
             "max_reserved": 5230.297088,
             "max_allocated": 5031.01696
         },
         "latency": {
             "unit": "s",
+            "mean": 6.015928203582767,
+            "stdev": 0.14041690444945987,
             "values": [
+                5.875511299133307,
+                6.156345108032227
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 42.38747394760056
         },
         "energy": null,
         "efficiency": null
         "memory": null,
         "latency": {
             "unit": "s",
+            "mean": 0.02359187530816772,
+            "stdev": 0.0015317100758449683,
             "values": [
+                0.02607417678833008,
+                0.024514495849609377,
+                0.023066751480102538,
+                0.022996896743774413,
+                0.02278704071044922,
+                0.022993919372558593,
+                0.02290176010131836,
+                0.022823936462402345,
+                0.022820863723754883,
+                0.02267955207824707,
+                0.022596704483032228,
+                0.022555551528930663,
                 0.02266009521484375,
+                0.022583295822143554,
+                0.022571008682250978,
+                0.022649856567382814,
+                0.02249113655090332,
                 0.022575103759765625,
+                0.022500352859497072,
+                0.022572032928466795,
+                0.02251571273803711,
+                0.02264678382873535,
+                0.022586368560791017,
+                0.022691839218139647,
+                0.022609920501708985,
+                0.02261203193664551,
+                0.022518720626831055,
+                0.022526975631713866,
                 0.022556671142578123,
+                0.022589439392089843,
                 0.022603776931762694,
+                0.02268569564819336,
+                0.02271129608154297,
+                0.022563840866088865,
+                0.022581247329711913,
+                0.022579200744628908,
+                0.022622207641601562,
                 0.022603776931762694,
+                0.022581247329711913,
+                0.022720672607421874,
+                0.022487903594970705,
+                0.02244607925415039,
+                0.02250752067565918,
+                0.02250444793701172,
+                0.02247372817993164,
+                0.022510591506958007,
+                0.02253004837036133,
+                0.022751232147216797,
                 0.022569984436035157,
+                0.022575103759765625,
                 0.022615039825439453,
                 0.02266111946105957,
+                0.02263039970397949,
+                0.022599679946899414,
                 0.022510591506958007,
+                0.022655168533325196,
+                0.02261894416809082,
+                0.0225167350769043,
+                0.02243791961669922,
+                0.022474720001220704,
+                0.022563840866088865,
                 0.02255462455749512,
+                0.022492383956909178,
+                0.0226844482421875,
+                0.022578176498413087,
+                0.022627328872680662,
+                0.022527999877929687,
+                0.022603776931762694,
+                0.02261299133300781,
+                0.022543359756469726,
+                0.022632448196411133,
+                0.02262835121154785,
+                0.022529184341430666,
+                0.02259542465209961,
+                0.022599775314331053,
+                0.022501279830932617,
+                0.02265497589111328,
+                0.022684959411621092,
+                0.022639328002929688,
+                0.022714368820190428,
+                0.022549503326416014,
                 0.022579200744628908,
+                0.022732799530029296,
+                0.022714368820190428,
+                0.022551551818847656,
+                0.023602176666259765,
+                0.024026111602783205,
+                0.0231014404296875,
+                0.024319999694824217,
+                0.02257529640197754,
+                0.02260870361328125,
+                0.022599807739257814,
+                0.022631359100341798,
+                0.02274502372741699,
+                0.022585344314575196,
                 0.022623231887817383,
                 0.022579200744628908,
+                0.02267238426208496,
+                0.022580223083496095,
+                0.022565887451171874,
+                0.02268057632446289,
+                0.02262015914916992,
+                0.022566911697387695,
+                0.02276675224304199,
+                0.022762336730957032,
+                0.022681600570678712,
+                0.02251263999938965,
+                0.022562816619873048,
+                0.022529024124145508,
+                0.0226396484375,
+                0.022568031311035155,
+                0.023253887176513673,
+                0.02433126449584961,
+                0.024397855758666993,
+                0.02452374458312988,
+                0.02555187225341797,
+                0.027206655502319335,
+                0.026950656890869142,
+                0.02732441520690918,
+                0.026236928939819337,
+                0.026479616165161132,
+                0.02656153678894043,
+                0.0267509765625,
+                0.026918912887573244,
+                0.026663936614990235,
+                0.026599424362182617,
+                0.02651852798461914,
+                0.02546998405456543,
+                0.023663583755493163,
+                0.02389811134338379,
+                0.022853919982910156,
+                0.02273865509033203,
+                0.022708223342895507,
+                0.022742015838623047,
+                0.0230830078125,
+                0.022776832580566408,
+                0.023129087448120117,
+                0.022749183654785156,
+                0.02267852783203125,
+                0.02273689651489258,
+                0.02292531204223633,
+                0.022737920761108397,
+                0.022619136810302733,
+                0.022581247329711913,
+                0.02265292739868164,
+                0.022730752944946288,
+                0.022569984436035157,
+                0.022590463638305663,
+                0.022855680465698244,
+                0.02270627212524414,
+                0.0226343994140625,
+                0.02264575958251953,
+                0.022587711334228516,
+                0.02254323196411133,
+                0.02259129524230957,
+                0.022540319442749025,
+                0.024703968048095704,
+                0.02484940719604492,
+                0.02478489685058594,
+                0.02495692825317383,
+                0.024859647750854492,
+                0.02479417610168457,
+                0.024747968673706055,
+                0.024995840072631836,
+                0.024964319229125977,
+                0.024827680587768554,
+                0.024827903747558593,
+                0.024785919189453123,
+                0.0249354248046875,
+                0.024936447143554686,
+                0.024892671585083008,
+                0.025030399322509767,
+                0.0231014404296875,
+                0.022625280380249024,
+                0.022543392181396484,
+                0.022599647521972657,
+                0.022564863204956053,
+                0.022566911697387695,
+                0.02249830436706543,
+                0.022517759323120116,
+                0.02246246337890625,
+                0.022429695129394533,
+                0.022544384002685547,
                 0.022527999877929687,
+                0.02255564880371094,
+                0.022675455093383787,
+                0.022609920501708985,
+                0.02253107261657715,
+                0.022626304626464845,
+                0.022570207595825197,
+                0.022565664291381834,
+                0.02253209686279297,
+                0.022544384002685547,
+                0.022714368820190428,
                 0.02271027183532715,
+                0.022617311477661134,
+                0.022630176544189452,
+                0.02263859176635742,
+                0.022529024124145508,
+                0.02257734489440918,
+                0.022954816818237304,
                 0.02267136001586914,
                 0.022692863464355468,
                 0.02253225517272949,
+                0.02261097526550293,
+                0.022647615432739257,
+                0.022585344314575196,
+                0.022579200744628908,
+                0.022567935943603516,
+                0.022545408248901368,
+                0.022617088317871094,
+                0.022626304626464845,
+                0.022519807815551757,
+                0.022603776931762694,
+                0.022578399658203126,
+                0.022635295867919923,
+                0.022786048889160155,
+                0.022606847763061523,
+                0.02253926467895508,
+                0.02268262481689453,
+                0.022601728439331056,
+                0.02266828727722168,
+                0.022599679946899414,
+                0.02266124725341797,
+                0.02254150390625,
+                0.02260652732849121,
+                0.02264371109008789,
+                0.022562816619873048,
+                0.022568960189819336,
+                0.022588415145874022,
+                0.022535423278808593,
+                0.02265983963012695,
+                0.023021568298339845,
                 0.022614015579223632,
+                0.022510591506958007,
+                0.022587392807006838,
+                0.022586591720581056,
+                0.022761247634887696,
+                0.022618112564086915,
+                0.02253209686279297,
+                0.022583391189575194,
+                0.022639520645141603,
+                0.022585344314575196,
                 0.022631423950195313,
+                0.022598655700683593,
+                0.022592512130737305,
+                0.02266828727722168,
+                0.02262348747253418,
+                0.022586111068725587,
+                0.023001087188720702,
+                0.02282598304748535,
+                0.022681600570678712,
                 0.022608896255493165,
+                0.022487071990966796,
+                0.022428640365600584,
+                0.023014175415039063,
+                0.02274406433105469,
+                0.022616064071655274,
+                0.022649856567382814,
+                0.02257027244567871,
+                0.022718175888061524,
+                0.02267852783203125,
+                0.022930431365966796,
+                0.02265936088562012,
+                0.02266908836364746,
+                0.02259654426574707,
+                0.02266214370727539,
+                0.022603776931762694,
+                0.02273587226867676,
+                0.022623231887817383,
+                0.025796607971191408,
+                0.030631935119628906,
+                0.024938495635986328,
+                0.02493235206604004,
+                0.02502374458312988,
+                0.024795904159545898,
+                0.02503987121582031,
+                0.024978431701660156,
+                0.025059423446655273,
+                0.02493145561218262,
+                0.025039648056030272,
+                0.02490572738647461,
+                0.025035776138305665,
+                0.024943616867065428,
+                0.024978464126586913,
+                0.02496227264404297,
+                0.02502835273742676,
+                0.024839168548583986,
+                0.025060352325439454,
+                0.025001983642578125,
+                0.02506342315673828,
+                0.02494492721557617,
+                0.024963808059692384,
+                0.02489049530029297,
+                0.024983423233032227,
+                0.024899904251098632,
+                0.02491667175292969,
+                0.024976383209228514,
+                0.02490572738647461,
+                0.024927295684814454,
+                0.024879039764404295,
+                0.022607872009277344,
+                0.022788095474243163,
+                0.02271027183532715,
+                0.022773759841918945,
+                0.02267750358581543,
+                0.022611967086791994,
+                0.022631423950195313,
+                0.02253948783874512,
+                0.02264147186279297,
+                0.022656991958618165,
+                0.022634496688842775,
+                0.022617088317871094,
+                0.022579200744628908,
+                0.022780927658081054,
+                0.022657024383544923,
+                0.022762496948242186,
+                0.02265190315246582,
+                0.02279033660888672,
+                0.022719295501708984,
+                0.022773759841918945,
+                0.022722848892211912,
+                0.022670047760009766,
+                0.0227061767578125,
+                0.022780927658081054,
+                0.022743040084838868,
+                0.022807680130004882,
+                0.022800256729125976,
+                0.022658239364624022,
+                0.022646591186523436,
+                0.02270310401916504,
+                0.022693151473999022,
+                0.022700767517089843,
+                0.022687744140625,
+                0.022866943359375,
+                0.022590463638305663,
+                0.02267865562438965,
+                0.024210304260253907,
+                0.02260905647277832,
+                0.02258518409729004,
+                0.022681600570678712,
                 0.02265907287597656,
+                0.022793407440185546,
+                0.022764352798461913,
+                0.022734943389892577,
+                0.022665119171142577,
+                0.022677696228027344,
+                0.02264556884765625,
+                0.022744224548339843,
+                0.022797151565551756,
+                0.02290073585510254,
+                0.02272051239013672,
+                0.02268671989440918,
+                0.022726655960083008,
+                0.02265395164489746,
+                0.022743040084838868,
+                0.02268569564819336,
+                0.022654144287109376,
+                0.022898496627807616,
+                0.02269081687927246,
+                0.022763519287109374,
+                0.022774879455566405,
+                0.022718528747558593,
+                0.02275209617614746,
+                0.022683647155761717,
+                0.022751232147216797,
+                0.022758399963378906,
+                0.02265497589111328,
+                0.022697984695434572,
+                0.02268671989440918,
+                0.022743040084838868,
+                0.02267136001586914,
+                0.02262118339538574,
+                0.02259996795654297,
+                0.02263632011413574,
+                0.02259244728088379,
+                0.022611967086791994,
+                0.02262015914916992,
+                0.022592512130737305,
+                0.022633472442626954,
+                0.022525951385498046,
+                0.022580223083496095,
+                0.022582271575927734,
+                0.02262835121154785,
+                0.02265190315246582,
+                0.022626367568969727,
+                0.022537151336669923,
+                0.02252288055419922,
+                0.022639616012573242,
+                0.02264678382873535,
+                0.022578176498413087,
+                0.022607872009277344,
+                0.022494207382202147,
+                0.022565887451171874,
+                0.022633663177490236,
+                0.02262124824523926,
+                0.022613759994506835,
+                0.02268057632446289,
+                0.022585344314575196,
+                0.022502399444580077,
+                0.02259660720825195,
+                0.022499359130859375,
+                0.022659263610839843,
+                0.022566688537597655,
+                0.022658048629760744,
+                0.022594560623168947,
+                0.022623231887817383,
+                0.022559743881225586,
+                0.02251366424560547,
+                0.022575103759765625,
+                0.02252288055419922,
+                0.022586368560791017,
+                0.02252390480041504,
+                0.02263145637512207,
+                0.022587360382080077,
+                0.02268876838684082,
+                0.022590463638305663,
                 0.022598655700683593,
+                0.02253107261657715,
+                0.022548479080200197,
+                0.02432512092590332,
+                0.022769664764404295,
+                0.022665216445922853,
+                0.022739967346191405,
+                0.022586368560791017,
+                0.02262937545776367,
+                0.022614015579223632,
+                0.023356416702270507,
+                0.025016319274902343,
+                0.02498371124267578,
+                0.02491097640991211,
+                0.025602783203125,
+                0.02657587242126465,
+                0.026410144805908205,
+                0.02645897674560547,
+                0.026505376815795897,
+                0.02640982437133789,
+                0.02639788818359375,
+                0.026466207504272463,
+                0.026440607070922852,
+                0.026440704345703125,
+                0.026418176651000977,
+                0.02646937561035156,
+                0.026415168762207033,
+                0.026463167190551758,
+                0.026359807968139647,
+                0.026267648696899414,
+                0.026449920654296875,
+                0.026390527725219725,
+                0.026422271728515623,
+                0.026463232040405273,
+                0.02648678398132324,
+                0.026462335586547852,
+                0.026481536865234374,
+                0.02653183937072754,
+                0.026371231079101564,
+                0.026352575302124023,
+                0.02630352020263672,
+                0.026426240921020507,
+                0.02655539131164551,
+                0.026589216232299803,
+                0.02651747131347656,
+                0.026489919662475585,
+                0.026378175735473634,
+                0.02633420753479004,
+                0.026426368713378907,
+                0.026474496841430665,
+                0.026393600463867187,
+                0.026418176651000977,
+                0.026400768280029296,
+                0.026372095108032227,
+                0.02631996726989746,
+                0.026346464157104493,
+                0.026364864349365233,
+                0.026444799423217775,
+                0.026394624710083008,
+                0.026432735443115234,
+                0.026223392486572267,
+                0.026407936096191405,
+                0.02643881607055664,
+                0.026346336364746092,
+                0.026447872161865234,
+                0.02631167984008789,
+                0.026351615905761717,
+                0.026432512283325195,
+                0.026392831802368164,
+                0.02630847930908203,
+                0.026290048599243165,
+                0.02634553527832031,
+                0.026405855178833006,
+                0.026345439910888672,
+                0.026302463531494142,
+                0.026395647048950196,
+                0.026366975784301756,
+                0.026232831954956053,
+                0.02629734420776367,
+                0.02628121566772461,
+                0.026434303283691406,
+                0.026398719787597655,
+                0.026335231781005858,
+                0.026487808227539062,
+                0.026403839111328126,
+                0.026331424713134766,
+                0.026326751708984374,
+                0.02630451202392578,
+                0.026383359909057616,
+                0.026281984329223632,
+                0.02632601547241211,
+                0.02447052764892578,
+                0.02266009521484375
             ]
         },
         "throughput": {
             "unit": "tokens/s",
+            "value": 42.387473947600554
         },
         "energy": null,
         "efficiency": null