Upload /gemma-7b/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

gemma-7b/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,755 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1076.453376,
+            "max_vram": 6796.869632,
+            "max_reserved": 6312.42752,
+            "max_allocated": 6191.16288
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.055446553140714006,
+            "stdev": 0.0003160784372994583,
+            "values": [
+                0.059538463592529296,
+                0.05553049468994141,
+                0.05571603012084961,
+                0.05579673767089844,
+                0.05584076690673828,
+                0.055626846313476565,
+                0.05526732635498047,
+                0.055311359405517575,
+                0.05532780838012695,
+                0.05533388900756836,
+                0.05530643081665039,
+                0.05528473663330078,
+                0.055311359405517575,
+                0.05533388900756836,
+                0.05530624008178711,
+                0.05529724884033203,
+                0.05537279891967774,
+                0.05535129547119141,
+                0.05535539245605469,
+                0.05533184051513672,
+                0.0553994255065918,
+                0.05529190444946289,
+                0.05543526458740235,
+                0.05533695983886719,
+                0.055413761138916016,
+                0.055314430236816405,
+                0.055375873565673826,
+                0.055349246978759765,
+                0.0553779182434082,
+                0.05532908630371094,
+                0.055365631103515625,
+                0.05534515380859375,
+                0.05537279891967774,
+                0.055624000549316405,
+                0.055360511779785154,
+                0.05535232162475586,
+                0.055349246978759765,
+                0.05537177658081055,
+                0.05555199813842773,
+                0.05565849685668945,
+                0.05556838226318359,
+                0.05551923370361328,
+                0.0555417594909668,
+                0.0556308479309082,
+                0.05552947235107422,
+                0.05540879821777344,
+                0.055554046630859374,
+                0.05562060928344727,
+                0.05536870574951172,
+                0.05544169616699219,
+                0.05543135833740234,
+                0.055446529388427736,
+                0.055422977447509764,
+                0.05543936157226562,
+                0.05540966415405273,
+                0.05557964706420898,
+                0.05539737701416016,
+                0.05547417449951172,
+                0.05535027313232422,
+                0.055416831970214846,
+                0.055393280029296874,
+                0.05540969467163086,
+                0.055433216094970705,
+                0.05543526458740235,
+                0.05540761566162109,
+                0.05538816070556641,
+                0.055419902801513675,
+                0.055384063720703126,
+                0.05542399978637695,
+                0.05540966415405273,
+                0.05538304138183594,
+                0.05542835235595703,
+                0.055398529052734374,
+                0.055376895904541014,
+                0.05543219375610352,
+                0.05533695983886719,
+                0.055400447845458986,
+                0.05542195129394531,
+                0.05540556716918945,
+                0.05542092895507812,
+                0.05542195129394531,
+                0.055411712646484375,
+                0.05536870574951172,
+                0.05542604827880859,
+                0.05539737701416016,
+                0.055446529388427736,
+                0.05545779037475586,
+                0.05543526458740235,
+                0.05538816070556641,
+                0.05538918304443359,
+                0.055428096771240234,
+                0.05543833541870117,
+                0.05547113418579101,
+                0.055428096771240234,
+                0.05541785430908203,
+                0.055376895904541014,
+                0.055449600219726565,
+                0.055430145263671876,
+                0.05537484741210937,
+                0.055406593322753904,
+                0.05545574569702148,
+                0.055462913513183595,
+                0.05545881652832031,
+                0.05542502212524414,
+                0.055462913513183595,
+                0.055422977447509764,
+                0.055373825073242185,
+                0.05540966415405273,
+                0.055444480895996094,
+                0.05540761566162109,
+                0.0553994255065918,
+                0.05537484741210937,
+                0.05544755172729492,
+                0.055695358276367186,
+                0.055588863372802735,
+                0.05535059356689453,
+                0.055392257690429686,
+                0.055395328521728515,
+                0.05539142227172852,
+                0.055444480895996094,
+                0.05539126586914062,
+                0.05536380767822266,
+                0.05539430236816406,
+                0.05540454483032226,
+                0.05536767959594727,
+                0.05547212982177734,
+                0.055436286926269535,
+                0.05545574569702148,
+                0.055346176147460936,
+                0.055395328521728515,
+                0.05537177658081055,
+                0.055362560272216796,
+                0.05536972808837891,
+                0.05538304138183594,
+                0.05541785430908203,
+                0.05537177658081055,
+                0.055392257690429686,
+                0.05544038391113281,
+                0.055419902801513675,
+                0.05545676803588867,
+                0.0554150390625,
+                0.0554312629699707,
+                0.05542707061767578,
+                0.05540966415405273,
+                0.055395328521728515,
+                0.055393280029296874,
+                0.055422046661376956,
+                0.055422977447509764,
+                0.055465984344482425,
+                0.05542399978637695,
+                0.055433311462402345,
+                0.05536153411865234,
+                0.055416831970214846,
+                0.05542195129394531,
+                0.05545574569702148,
+                0.05543423843383789,
+                0.05543526458740235,
+                0.055393280029296874,
+                0.055414783477783204,
+                0.05547020721435547,
+                0.05538816070556641,
+                0.05546188735961914,
+                0.05556121444702149,
+                0.055422977447509764,
+                0.05541785430908203,
+                0.05541888046264649,
+                0.05545471954345703,
+                0.05543526458740235,
+                0.055419902801513675,
+                0.05542195129394531,
+                0.05543219375610352,
+                0.055392513275146486,
+                0.05549363327026367,
+                0.05542911911010742,
+                0.05532467269897461,
+                0.055429374694824216,
+                0.05541888046264649,
+                0.05543423843383789,
+                0.05545574569702148,
+                0.05544243240356445,
+                0.05536671829223633
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 4617.0588701937
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1076.453376,
+            "max_vram": 7212.105728,
+            "max_reserved": 6727.663616,
+            "max_allocated": 6570.75456
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 8.033202688217166,
+            "stdev": 0.007208446502684929,
+            "values": [
+                8.02599424171448,
+                8.04041113471985
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 31.743254825877294
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.03150275564006731,
+            "stdev": 0.00016209291603310047,
+            "values": [
+                0.031562751770019534,
+                0.031319040298461914,
+                0.03144396781921387,
+                0.031682559967041016,
+                0.03152383995056152,
+                0.0315996150970459,
+                0.03178291130065918,
+                0.03163340759277344,
+                0.03158118438720703,
+                0.031475711822509765,
+                0.03146240043640137,
+                0.03150028800964356,
+                0.031528959274291994,
+                0.03155148887634277,
+                0.03143987274169922,
+                0.03138457679748535,
+                0.03148595237731933,
+                0.03145113563537598,
+                0.03146854400634766,
+                0.031476736068725586,
+                0.03157811164855957,
+                0.03155251121520996,
+                0.03141222381591797,
+                0.03153715133666992,
+                0.03138662338256836,
+                0.03162931251525879,
+                0.0315361270904541,
+                0.0316364803314209,
+                0.031508480072021484,
+                0.031545343399047854,
+                0.03149619293212891,
+                0.03170508766174317,
+                0.032023551940917966,
+                0.031527936935424807,
+                0.031357952117919925,
+                0.031442943572998046,
+                0.03146854400634766,
+                0.031509504318237305,
+                0.03151379203796387,
+                0.031389503479003905,
+                0.031357952117919925,
+                0.03146444892883301,
+                0.0314337272644043,
+                0.031410175323486327,
+                0.03131084823608398,
+                0.031437824249267575,
+                0.031748096466064454,
+                0.031937536239624024,
+                0.03153919982910156,
+                0.03128934478759766,
+                0.03133235168457031,
+                0.03137433624267578,
+                0.03139583969116211,
+                0.03155251121520996,
+                0.031459327697753905,
+                0.031497215270996096,
+                0.03158732795715332,
+                0.03143577575683594,
+                0.03140403175354004,
+                0.03129855918884277,
+                0.031307775497436525,
+                0.03138559913635254,
+                0.03133440017700195,
+                0.03139072036743164,
+                0.03126169586181641,
+                0.03149926376342774,
+                0.03137638473510742,
+                0.031542272567749025,
+                0.031527936935424807,
+                0.03144499206542969,
+                0.03144508743286133,
+                0.031474592208862305,
+                0.03147776031494141,
+                0.03150643157958984,
+                0.03136000061035156,
+                0.03140300750732422,
+                0.03158016014099121,
+                0.03146137619018555,
+                0.03141529655456543,
+                0.03130982398986817,
+                0.031275007247924806,
+                0.03125043106079101,
+                0.03142963218688965,
+                0.03153817558288574,
+                0.03127910423278808,
+                0.03137433624267578,
+                0.031541248321533204,
+                0.031494144439697266,
+                0.031437984466552736,
+                0.031232864379882812,
+                0.031268863677978515,
+                0.03138559913635254,
+                0.031349920272827146,
+                0.03145302391052246,
+                0.03134771156311035,
+                0.03134873580932617,
+                0.031939584732055666,
+                0.03139481544494629,
+                0.0313734073638916,
+                0.03137731170654297,
+                0.03143680000305176,
+                0.03138662338256836,
+                0.03137638473510742,
+                0.031476736068725586,
+                0.0314019832611084,
+                0.03143270492553711,
+                0.03145856094360352,
+                0.031394559860229494,
+                0.03144499206542969,
+                0.03141632080078125,
+                0.03139788818359375,
+                0.03163033676147461,
+                0.0314654712677002,
+                0.03143168067932129,
+                0.03123200035095215,
+                0.031270912170410156,
+                0.031381727218627926,
+                0.03144272041320801,
+                0.0314553279876709,
+                0.03141007995605469,
+                0.03135078430175781,
+                0.03142451286315918,
+                0.031324159622192385,
+                0.03141734313964844,
+                0.031455232620239255,
+                0.03136000061035156,
+                0.03151667213439941,
+                0.031389696121215824,
+                0.031438848495483396,
+                0.03139072036743164,
+                0.0313118724822998,
+                0.031319040298461914,
+                0.031373311996459964,
+                0.03125555229187012,
+                0.031352832794189454,
+                0.031335424423217774,
+                0.031513599395751955,
+                0.03142860794067383,
+                0.03161395263671875,
+                0.03122790336608887,
+                0.0313753604888916,
+                0.03139174461364746,
+                0.03144704055786133,
+                0.03131391906738281,
+                0.03155660820007324,
+                0.03143475151062012,
+                0.0314019832611084,
+                0.03137843132019043,
+                0.03141939163208008,
+                0.031290367126464845,
+                0.03128934478759766,
+                0.03134976005554199,
+                0.03144806480407715,
+                0.03136614418029785,
+                0.03134668731689453,
+                0.03143270492553711,
+                0.03141750335693359,
+                0.031449951171875,
+                0.03149619293212891,
+                0.0313753604888916,
+                0.03191193580627441,
+                0.03205734252929687,
+                0.031541248321533204,
+                0.031426559448242186,
+                0.03150540733337402,
+                0.03137433624267578,
+                0.0314738883972168,
+                0.03138435173034668,
+                0.031406080245971676,
+                0.03145113563537598,
+                0.03143680000305176,
+                0.031475711822509765,
+                0.03139379119873047,
+                0.03144396781921387,
+                0.031545343399047854,
+                0.03150028800964356,
+                0.03200102233886719,
+                0.03157094383239746,
+                0.031604736328125,
+                0.03160166358947754,
+                0.0315729923248291,
+                0.03158118438720703,
+                0.03149926376342774,
+                0.031302656173706055,
+                0.03139276885986328,
+                0.031373311996459964,
+                0.0313702392578125,
+                0.031341567993164066,
+                0.03138457679748535,
+                0.03139379119873047,
+                0.031297536849975584,
+                0.03133235168457031,
+                0.031324159622192385,
+                0.0313702392578125,
+                0.03141632080078125,
+                0.031410175323486327,
+                0.03155046463012695,
+                0.031437824249267575,
+                0.031362079620361326,
+                0.03139478492736816,
+                0.03150643157958984,
+                0.031474687576293944,
+                0.031460351943969726,
+                0.03162009620666504,
+                0.03156070327758789,
+                0.03148287963867188,
+                0.031308799743652346,
+                0.031246335983276367,
+                0.03131084823608398,
+                0.0315361270904541,
+                0.03142860794067383,
+                0.031531007766723636,
+                0.031546367645263675,
+                0.03146137619018555,
+                0.03149926376342774,
+                0.031508480072021484,
+                0.031543296813964845,
+                0.03150233650207519,
+                0.031503360748291014,
+                0.03156684875488281,
+                0.03148819160461426,
+                0.03152262306213379,
+                0.03336819076538086,
+                0.032889728546142576,
+                0.03186073684692383,
+                0.031715328216552735,
+                0.031459327697753905,
+                0.03138764762878418,
+                0.031480831146240236,
+                0.03141119956970215,
+                0.03152297592163086,
+                0.031357791900634764,
+                0.0314521598815918,
+                0.0313753604888916,
+                0.03153408050537109,
+                0.031527936935424807,
+                0.03159552001953125,
+                0.03146342468261719,
+                0.03151155281066895,
+                0.03140505599975586,
+                0.03172147178649903,
+                0.03139174461364746,
+                0.03139993667602539,
+                0.03142348861694336,
+                0.03156377601623535,
+                0.03147878456115723,
+                0.03146751976013184,
+                0.03139174461364746,
+                0.031421440124511715,
+                0.0315545597076416,
+                0.0315729923248291,
+                0.03163340759277344,
+                0.03156800079345703,
+                0.031532928466796874,
+                0.03162931251525879,
+                0.03143270492553711,
+                0.03142963218688965,
+                0.03137638473510742,
+                0.031455232620239255,
+                0.03148902320861816,
+                0.031545343399047854,
+                0.03138355255126953,
+                0.03133030319213867,
+                0.031238143920898437,
+                0.031514623641967776,
+                0.031425535202026365,
+                0.031480831146240236,
+                0.03136511993408203,
+                0.0316231689453125,
+                0.03144396781921387,
+                0.03144499206542969,
+                0.03142860794067383,
+                0.03159244728088379,
+                0.03155660820007324,
+                0.03176755142211914,
+                0.031524864196777344,
+                0.03155353546142578,
+                0.03145030403137207,
+                0.03153900718688965,
+                0.03141734313964844,
+                0.03157708740234375,
+                0.031426559448242186,
+                0.03162623977661133,
+                0.03157827186584473,
+                0.03164143943786621,
+                0.03159654426574707,
+                0.0317573127746582,
+                0.03158527946472168,
+                0.031850496292114255,
+                0.0317071361541748,
+                0.031733760833740236,
+                0.03161497688293457,
+                0.031696895599365234,
+                0.0315361270904541,
+                0.0315729923248291,
+                0.03158835220336914,
+                0.031492095947265625,
+                0.031488000869750975,
+                0.031661056518554685,
+                0.03157401657104492,
+                0.031542272567749025,
+                0.03148518371582031,
+                0.03188096046447754,
+                0.03157094383239746,
+                0.03162623977661133,
+                0.03158016014099121,
+                0.03140403175354004,
+                0.03141529655456543,
+                0.03162623977661133,
+                0.03152383995056152,
+                0.03159244728088379,
+                0.03144806480407715,
+                0.031455232620239255,
+                0.03163852882385254,
+                0.03159756851196289,
+                0.03147878456115723,
+                0.03149004745483398,
+                0.03143270492553711,
+                0.031627264022827145,
+                0.03160985565185547,
+                0.03161702346801758,
+                0.031498239517211916,
+                0.03139276885986328,
+                0.03156480026245117,
+                0.031493120193481446,
+                0.03156991958618164,
+                0.03146444892883301,
+                0.031508480072021484,
+                0.031507455825805664,
+                0.031720447540283206,
+                0.03151974487304687,
+                0.03137638473510742,
+                0.03143680000305176,
+                0.03146342468261719,
+                0.03141222381591797,
+                0.03136000061035156,
+                0.03153408050537109,
+                0.03138150405883789,
+                0.03138252830505371,
+                0.031321088790893556,
+                0.031438848495483396,
+                0.031488000869750975,
+                0.031526912689208986,
+                0.03151667213439941,
+                0.0315156478881836,
+                0.03196108818054199,
+                0.03144908714294434,
+                0.03153817558288574,
+                0.03150131225585937,
+                0.03153919982910156,
+                0.03196006393432617,
+                0.03161702346801758,
+                0.031457279205322264,
+                0.0316753921508789,
+                0.031492095947265625,
+                0.031506624221801754,
+                0.031460159301757815,
+                0.03161718368530273,
+                0.03165679931640625,
+                0.03157606315612793,
+                0.03151769638061523,
+                0.03153932762145996,
+                0.031587200164794924,
+                0.03156684875488281,
+                0.031470592498779294,
+                0.03139583969116211,
+                0.031867904663085936,
+                0.03131494331359863,
+                0.03156889533996582,
+                0.031508480072021484,
+                0.031454208374023435,
+                0.03155251121520996,
+                0.0315545597076416,
+                0.03156070327758789,
+                0.03149516868591309,
+                0.03158527946472168,
+                0.031591424942016604,
+                0.031528959274291994,
+                0.0316682243347168,
+                0.03153817558288574,
+                0.031664127349853514,
+                0.031491071701049804,
+                0.03153510475158691,
+                0.031503360748291014,
+                0.031476736068725586,
+                0.031458303451538085,
+                0.0315996150970459,
+                0.03159347152709961,
+                0.03163443183898926,
+                0.03151769638061523,
+                0.03159654426574707,
+                0.03153203201293946,
+                0.03143475151062012,
+                0.03160172843933105,
+                0.03174496078491211,
+                0.03150131225585937,
+                0.0315545597076416,
+                0.03158835220336914,
+                0.03158323287963867,
+                0.031541248321533204,
+                0.03152179145812988,
+                0.03144499206542969,
+                0.0315228157043457,
+                0.03149619293212891,
+                0.03153203201293946,
+                0.031615167617797854,
+                0.03149190330505371,
+                0.03146751976013184,
+                0.031527936935424807,
+                0.03151769638061523,
+                0.031493120193481446,
+                0.0316180477142334,
+                0.03141529655456543,
+                0.031663103103637694,
+                0.03196723175048828,
+                0.03158527946472168,
+                0.031547391891479495,
+                0.03149926376342774,
+                0.03157401657104492,
+                0.03150028800964356,
+                0.03179110336303711,
+                0.03153305625915527,
+                0.03141427230834961,
+                0.031632383346557616,
+                0.031664127349853514,
+                0.03160576057434082,
+                0.0316682243347168,
+                0.03164057540893555,
+                0.03158220863342285,
+                0.031748096466064454,
+                0.031719423294067385,
+                0.031545343399047854,
+                0.03150540733337402,
+                0.03156172752380371,
+                0.031426559448242186,
+                0.03152297592163086,
+                0.03153801536560059,
+                0.0314204158782959,
+                0.031510528564453126,
+                0.03144908714294434,
+                0.03160268783569336,
+                0.03149231910705566,
+                0.03153078460693359,
+                0.03153305625915527,
+                0.031494144439697266,
+                0.03153203201293946,
+                0.031529983520507815,
+                0.03144499206542969,
+                0.03158937644958496,
+                0.03160371208190918,
+                0.031645696640014646,
+                0.03155660820007324,
+                0.031527936935424807,
+                0.03160371208190918,
+                0.03149516868591309,
+                0.031512575149536134,
+                0.03143475151062012,
+                0.03146649551391602,
+                0.031615999221801756,
+                0.03143475151062012,
+                0.03144704055786133,
+                0.03146854400634766,
+                0.03150028800964356,
+                0.03146649551391602,
+                0.031508480072021484,
+                0.0314726390838623,
+                0.031422464370727536,
+                0.03139686393737793,
+                0.031426559448242186,
+                0.03145017623901367,
+                0.03149407958984375,
+                0.031459327697753905,
+                0.03146956825256347,
+                0.03159859275817871,
+                0.0317071361541748,
+                0.031438848495483396,
+                0.031388671875,
+                0.03137446403503418,
+                0.03176435279846192,
+                0.03150643157958984,
+                0.03153408050537109,
+                0.03131391906738281,
+                0.0313384952545166,
+                0.03136716842651367,
+                0.03156070327758789,
+                0.03142758369445801,
+                0.03144499206542969,
+                0.03146956825256347,
+                0.03154841613769531,
+                0.031493120193481446,
+                0.031491071701049804,
+                0.031508480072021484,
+                0.031509504318237305,
+                0.0316376953125,
+                0.0316055679321289,
+                0.03156991958618164,
+                0.031480831146240236,
+                0.031437824249267575,
+                0.03150643157958984,
+                0.03157401657104492,
+                0.031438848495483396,
+                0.03140096092224121,
+                0.03137638473510742,
+                0.031494144439697266,
+                0.03158732795715332,
+                0.031497215270996096,
+                0.031510528564453126,
+                0.03137740707397461,
+                0.031421440124511715,
+                0.031422464370727536
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 31.743254825877298
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}