Upload /gemma-2b/int8_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

gemma-2b/int8_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,837 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1412.03456,
+            "max_vram": 4083.154944,
+            "max_reserved": 3590.324224,
+            "max_allocated": 3509.052928
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.03808782375901371,
+            "stdev": 0.0005095311012424405,
+            "values": [
+                0.04125299072265625,
+                0.03919974517822265,
+                0.03892531204223633,
+                0.03883520126342774,
+                0.038712318420410154,
+                0.038934528350830076,
+                0.042646526336669925,
+                0.04177203369140625,
+                0.03861094284057617,
+                0.03809996795654297,
+                0.03808051300048828,
+                0.03812351989746094,
+                0.037966846466064456,
+                0.037907455444335936,
+                0.037819393157958986,
+                0.03811328125,
+                0.037928958892822266,
+                0.03782144165039063,
+                0.03791360092163086,
+                0.03848294448852539,
+                0.037779457092285154,
+                0.037973121643066404,
+                0.03802316665649414,
+                0.03797708892822266,
+                0.037945343017578126,
+                0.03803238296508789,
+                0.03797196960449219,
+                0.037986305236816405,
+                0.03821875381469726,
+                0.0381317138671875,
+                0.03812454223632813,
+                0.03812761688232422,
+                0.038421504974365236,
+                0.0381030387878418,
+                0.03810201644897461,
+                0.03809280014038086,
+                0.038198272705078126,
+                0.03812659072875976,
+                0.03818393707275391,
+                0.03810115051269531,
+                0.038125568389892575,
+                0.0379156494140625,
+                0.037792766571044925,
+                0.037874687194824216,
+                0.03790028762817383,
+                0.03782451248168945,
+                0.03783987045288086,
+                0.03784499359130859,
+                0.03781324768066406,
+                0.0379791374206543,
+                0.03798015975952149,
+                0.03793612670898437,
+                0.03796377563476563,
+                0.037797889709472655,
+                0.03777151870727539,
+                0.03784601593017578,
+                0.03791872024536133,
+                0.03807743835449219,
+                0.03792486572265625,
+                0.03782144165039063,
+                0.03780198287963867,
+                0.03785113525390625,
+                0.03782656097412109,
+                0.03785420989990235,
+                0.03779072189331055,
+                0.037951488494873044,
+                0.03807743835449219,
+                0.03796377563476563,
+                0.03787571334838867,
+                0.037997566223144534,
+                0.037967872619628903,
+                0.037959678649902344,
+                0.0385351676940918,
+                0.03802009582519531,
+                0.03817881774902344,
+                0.03800371170043945,
+                0.03801190567016602,
+                0.03805187225341797,
+                0.03799244689941406,
+                0.03788083267211914,
+                0.037981182098388674,
+                0.037996543884277346,
+                0.03780198287963867,
+                0.038150142669677735,
+                0.03806412887573242,
+                0.03828736114501953,
+                0.03803647994995117,
+                0.038591487884521485,
+                0.04000153732299805,
+                0.03995340728759766,
+                0.039180286407470705,
+                0.03784710311889648,
+                0.037874687194824216,
+                0.03793407821655274,
+                0.03789823913574219,
+                0.03800064086914062,
+                0.03801190567016602,
+                0.03806316757202149,
+                0.03803238296508789,
+                0.037988353729248046,
+                0.0379422721862793,
+                0.03791257476806641,
+                0.03780710220336914,
+                0.03803852844238281,
+                0.038209537506103515,
+                0.038004737854003906,
+                0.03812659072875976,
+                0.03807846450805664,
+                0.037975135803222655,
+                0.038166526794433595,
+                0.037917697906494144,
+                0.037943294525146484,
+                0.03786963272094727,
+                0.037872638702392575,
+                0.03785318374633789,
+                0.03785420989990235,
+                0.03796889495849609,
+                0.03778252792358398,
+                0.037891071319580076,
+                0.03787776184082031,
+                0.03796582412719727,
+                0.038075393676757815,
+                0.03802316665649414,
+                0.038084606170654296,
+                0.03801804733276367,
+                0.03852799987792969,
+                0.03808870315551758,
+                0.03809791946411133,
+                0.03818188858032227,
+                0.03801804733276367,
+                0.037991424560546876,
+                0.0380211181640625,
+                0.037978111267089845,
+                0.038171646118164065,
+                0.038117374420166016,
+                0.038179840087890625,
+                0.03794636917114258,
+                0.0380335693359375,
+                0.038286334991455076,
+                0.037905406951904294,
+                0.0381214714050293,
+                0.03812044906616211,
+                0.03798425674438476,
+                0.03788185501098633,
+                0.03786547088623047,
+                0.038059009552001956,
+                0.03787776184082031,
+                0.037951488494873044,
+                0.037873729705810544,
+                0.03784812927246094,
+                0.03780198287963867,
+                0.03791072082519531,
+                0.03786547088623047,
+                0.03784499359130859,
+                0.037855232238769534,
+                0.03792486572265625,
+                0.037940223693847655,
+                0.037953536987304685,
+                0.03784806442260742,
+                0.038011104583740234,
+                0.03788083267211914,
+                0.037996543884277346,
+                0.037989376068115234,
+                0.03799552154541016,
+                0.03794739151000977,
+                0.03795878219604492,
+                0.03784220886230469,
+                0.03803340911865234,
+                0.038059009552001956,
+                0.03802828979492188,
+                0.03803852844238281,
+                0.03793920135498047,
+                0.03797094345092773,
+                0.03793612670898437,
+                0.03791155242919922,
+                0.037972991943359374,
+                0.038021278381347653,
+                0.0381399040222168,
+                0.03840921783447265,
+                0.03800678253173828,
+                0.037989376068115234,
+                0.03797196960449219,
+                0.0379249267578125,
+                0.037910526275634765,
+                0.03792281723022461,
+                0.037961727142333986,
+                0.0379504623413086,
+                0.03792998504638672,
+                0.037975040435791016,
+                0.038002689361572264,
+                0.037991424560546876,
+                0.03832729721069336,
+                0.03796275329589844,
+                0.03797094345092773,
+                0.03808870315551758,
+                0.03810611343383789,
+                0.03803340911865234,
+                0.03803852844238281,
+                0.038019073486328124,
+                0.03805081558227539,
+                0.03794944000244141,
+                0.03798323059082031,
+                0.03801599884033203,
+                0.03802009582519531,
+                0.03793407821655274,
+                0.03798015975952149,
+                0.037937152862548826,
+                0.03795558547973633,
+                0.038125568389892575,
+                0.0381030387878418,
+                0.03813580703735352,
+                0.038168575286865236,
+                0.03804467010498047,
+                0.03808153533935547,
+                0.03805081558227539,
+                0.038091777801513675,
+                0.03805491256713867,
+                0.03797196960449219,
+                0.038019073486328124,
+                0.03790643310546875,
+                0.037904384613037106,
+                0.03791155242919922,
+                0.03800166320800781,
+                0.03797708892822266,
+                0.03796377563476563,
+                0.0377968635559082,
+                0.03785830307006836,
+                0.03800371170043945,
+                0.037981182098388674,
+                0.03803238296508789,
+                0.03784806442260742,
+                0.038242305755615234,
+                0.037855232238769534,
+                0.03787059020996094,
+                0.0378152961730957,
+                0.037751808166503906,
+                0.037850112915039064,
+                0.037823486328125,
+                0.03791360092163086,
+                0.038201534271240234,
+                0.040304641723632816,
+                0.03809491348266601,
+                0.03794739151000977,
+                0.038042625427246096,
+                0.038441982269287106,
+                0.03798220825195313,
+                0.037891071319580076,
+                0.037951488494873044,
+                0.038019073486328124,
+                0.037928958892822266,
+                0.037923839569091795,
+                0.03784214401245117,
+                0.03791155242919922,
+                0.038075393676757815,
+                0.03788595199584961,
+                0.03799039840698242,
+                0.03792281723022461,
+                0.038152191162109376,
+                0.037946529388427734,
+                0.03805081558227539,
+                0.037994495391845705,
+                0.03806924819946289,
+                0.03803852844238281
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 6721.308143509146
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1412.03456,
+            "max_vram": 4359.979008,
+            "max_reserved": 3867.148288,
+            "max_allocated": 3775.922688
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 9.541718002319335,
+            "stdev": 0.007792644500730361,
+            "values": [
+                9.533925357818605,
+                9.549510646820066
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 26.72474704639315
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.037418501969879774,
+            "stdev": 0.0001500157972494668,
+            "values": [
+                0.0373493766784668,
+                0.03758899307250976,
+                0.03731148910522461,
+                0.03736678314208984,
+                0.03735859298706055,
+                0.03740262222290039,
+                0.03726438522338867,
+                0.03727052688598633,
+                0.037285888671875,
+                0.03730124664306641,
+                0.03715379333496094,
+                0.03732992172241211,
+                0.03722956848144531,
+                0.037259391784667965,
+                0.037182334899902345,
+                0.03743539047241211,
+                0.03743231964111328,
+                0.03750912094116211,
+                0.037440513610839846,
+                0.03736064147949219,
+                0.037318656921386716,
+                0.037806079864501956,
+                0.03740364837646484,
+                0.03748556900024414,
+                0.03747020721435547,
+                0.03747750473022461,
+                0.03748851013183594,
+                0.0374466552734375,
+                0.03739340972900391,
+                0.037354496002197264,
+                0.03743231964111328,
+                0.0375142707824707,
+                0.03738211059570312,
+                0.03747123336791992,
+                0.03765657424926758,
+                0.03754086303710937,
+                0.03733299255371094,
+                0.03730944061279297,
+                0.03724288177490234,
+                0.03729100799560547,
+                0.03723980712890625,
+                0.03728691101074219,
+                0.03728793716430664,
+                0.03743027114868164,
+                0.03728486251831055,
+                0.0373493766784668,
+                0.037207038879394534,
+                0.037303295135498044,
+                0.03738214492797851,
+                0.037259391784667965,
+                0.03724070358276367,
+                0.03719475173950195,
+                0.03711795043945312,
+                0.03729510498046875,
+                0.03724800109863281,
+                0.03743027114868164,
+                0.03732889556884766,
+                0.03723484802246094,
+                0.03722121429443359,
+                0.03726131057739258,
+                0.037275646209716795,
+                0.03726847839355469,
+                0.037256191253662106,
+                0.03739561462402344,
+                0.03733385467529297,
+                0.037381118774414065,
+                0.0374835205078125,
+                0.03744873428344726,
+                0.03740975952148438,
+                0.037369857788085936,
+                0.03742617416381836,
+                0.03736064147949219,
+                0.0373493766784668,
+                0.03731148910522461,
+                0.037400577545166014,
+                0.037784576416015625,
+                0.03732992172241211,
+                0.0372408332824707,
+                0.03726028823852539,
+                0.03742924880981445,
+                0.03743436813354492,
+                0.03744063949584961,
+                0.03727654266357422,
+                0.03723263931274414,
+                0.03731353759765625,
+                0.03721318435668945,
+                0.037292030334472655,
+                0.03736479949951172,
+                0.03761350250244141,
+                0.03743033599853516,
+                0.03734828948974609,
+                0.03733299255371094,
+                0.037362686157226564,
+                0.037526527404785154,
+                0.037400577545166014,
+                0.037294078826904296,
+                0.03729715347290039,
+                0.03733401489257813,
+                0.03729510498046875,
+                0.037305343627929685,
+                0.03729305648803711,
+                0.03734543991088867,
+                0.03735228729248047,
+                0.03740671920776367,
+                0.037321727752685545,
+                0.03724800109863281,
+                0.03729305648803711,
+                0.037372928619384765,
+                0.03743436813354492,
+                0.03746713638305664,
+                0.037400577545166014,
+                0.03745792007446289,
+                0.03742822265625,
+                0.037395454406738284,
+                0.0374200325012207,
+                0.03735551834106445,
+                0.03733913421630859,
+                0.037340160369873046,
+                0.03729510498046875,
+                0.037356544494628906,
+                0.03739340972900391,
+                0.03742924880981445,
+                0.0373309440612793,
+                0.03733196640014649,
+                0.03727360153198242,
+                0.0373309440612793,
+                0.037343231201171875,
+                0.037321727752685545,
+                0.037215232849121094,
+                0.037766143798828124,
+                0.037288959503173826,
+                0.0373831672668457,
+                0.03726131057739258,
+                0.03713433456420898,
+                0.037408767700195314,
+                0.037318656921386716,
+                0.03739136123657227,
+                0.037312702178955076,
+                0.03725088119506836,
+                0.03748681640625,
+                0.03748124694824219,
+                0.03768012619018555,
+                0.03748761749267578,
+                0.03728998565673828,
+                0.03718656158447266,
+                0.03726438522338867,
+                0.03726540756225586,
+                0.037335041046142575,
+                0.03745177459716797,
+                0.037275646209716795,
+                0.037305343627929685,
+                0.037338111877441404,
+                0.03739136123657227,
+                0.03757056045532227,
+                0.03741593551635742,
+                0.03737500762939453,
+                0.037402591705322265,
+                0.037176319122314457,
+                0.03731660842895508,
+                0.037341182708740234,
+                0.037185535430908204,
+                0.03752256011962891,
+                0.0373963508605957,
+                0.03736576080322265,
+                0.037275646209716795,
+                0.03739340972900391,
+                0.0373493766784668,
+                0.03746815872192383,
+                0.0372305908203125,
+                0.037253120422363284,
+                0.037482494354248046,
+                0.03727872085571289,
+                0.03732582473754883,
+                0.03736678314208984,
+                0.03743743896484375,
+                0.037531646728515625,
+                0.03738828659057617,
+                0.037253120422363284,
+                0.037323776245117186,
+                0.037106689453125,
+                0.03730636978149414,
+                0.037367809295654295,
+                0.037233665466308595,
+                0.0378787841796875,
+                0.037340160369873046,
+                0.03752153778076172,
+                0.03754790496826172,
+                0.037407745361328126,
+                0.037356544494628906,
+                0.03724185562133789,
+                0.037525505065917966,
+                0.03742924880981445,
+                0.03733913421630859,
+                0.03743436813354492,
+                0.03740467071533203,
+                0.037910526275634765,
+                0.037610496520996094,
+                0.0375623664855957,
+                0.03747635269165039,
+                0.037610496520996094,
+                0.037602302551269534,
+                0.03764223861694336,
+                0.03752140808105469,
+                0.037372928619384765,
+                0.03741388702392578,
+                0.03748863983154297,
+                0.03739340972900391,
+                0.03738623809814453,
+                0.0373196792602539,
+                0.03734732818603516,
+                0.03746201705932617,
+                0.03734527969360352,
+                0.03739852905273437,
+                0.03731558227539063,
+                0.03748863983154297,
+                0.03740262222290039,
+                0.03738521575927734,
+                0.03729612731933594,
+                0.0375470085144043,
+                0.037397502899169925,
+                0.03738214492797851,
+                0.03743641662597656,
+                0.03826483154296875,
+                0.03727974319458008,
+                0.03724492645263672,
+                0.037479423522949216,
+                0.03736371231079102,
+                0.037323776245117186,
+                0.03727974319458008,
+                0.037410816192626956,
+                0.0373493766784668,
+                0.03739033508300781,
+                0.03736678314208984,
+                0.03767398452758789,
+                0.037599231719970705,
+                0.037422080993652344,
+                0.037803009033203126,
+                0.0375203857421875,
+                0.03761151885986328,
+                0.03734630584716797,
+                0.037253120422363284,
+                0.037425151824951174,
+                0.03745382308959961,
+                0.037465087890625,
+                0.037738494873046875,
+                0.03761151885986328,
+                0.03749990463256836,
+                0.037449726104736326,
+                0.03757875061035156,
+                0.037901313781738284,
+                0.03750092697143555,
+                0.037305343627929685,
+                0.037182464599609374,
+                0.03726233673095703,
+                0.03725414276123047,
+                0.037407745361328126,
+                0.037338111877441404,
+                0.03749478530883789,
+                0.037544960021972655,
+                0.03721932983398438,
+                0.03738623809814453,
+                0.03736576080322265,
+                0.037142559051513674,
+                0.03720495986938477,
+                0.037424129486083986,
+                0.03729305648803711,
+                0.03737702560424805,
+                0.03746307373046875,
+                0.03760022354125977,
+                0.03743334579467773,
+                0.037548030853271484,
+                0.03746918487548828,
+                0.03769548797607422,
+                0.0373196792602539,
+                0.03733299255371094,
+                0.03744255828857422,
+                0.03747430419921875,
+                0.03760332870483398,
+                0.037394432067871096,
+                0.03748659133911133,
+                0.037550079345703126,
+                0.0376627197265625,
+                0.037678081512451174,
+                0.037648384094238284,
+                0.037572734832763674,
+                0.03763187026977539,
+                0.037556224822998044,
+                0.03753062438964844,
+                0.03750604629516602,
+                0.03801702499389648,
+                0.03744460678100586,
+                0.03752959823608398,
+                0.03751628875732422,
+                0.03769241714477539,
+                0.03739955139160156,
+                0.03742617416381836,
+                0.03746918487548828,
+                0.03738521575927734,
+                0.037428256988525394,
+                0.03741692733764648,
+                0.03734124755859375,
+                0.037350334167480466,
+                0.037628929138183595,
+                0.03718143844604492,
+                0.03743641662597656,
+                0.0373196792602539,
+                0.03733708953857422,
+                0.03733708953857422,
+                0.03729612731933594,
+                0.03744870376586914,
+                0.03742617416381836,
+                0.037544960021972655,
+                0.03751424026489258,
+                0.037599231719970705,
+                0.03754393768310547,
+                0.03735756683349609,
+                0.03744870376586914,
+                0.03721420669555664,
+                0.03745894241333008,
+                0.037379070281982424,
+                0.03716934585571289,
+                0.03743315124511719,
+                0.03728691101074219,
+                0.03747532653808594,
+                0.0373043212890625,
+                0.03732070541381836,
+                0.037378047943115236,
+                0.037561344146728515,
+                0.03741798400878906,
+                0.03749478530883789,
+                0.037405696868896485,
+                0.03737190246582031,
+                0.03739648056030274,
+                0.03750297546386719,
+                0.03750502395629883,
+                0.03729817581176758,
+                0.03751321411132812,
+                0.03744255828857422,
+                0.03731062316894531,
+                0.03737276840209961,
+                0.03758703994750977,
+                0.03739433670043945,
+                0.03751731109619141,
+                0.03823820877075195,
+                0.03770982360839844,
+                0.03751833724975586,
+                0.03739852905273437,
+                0.03749273681640625,
+                0.037427200317382815,
+                0.03728998565673828,
+                0.03747225570678711,
+                0.03738726425170898,
+                0.03736371231079102,
+                0.0374835205078125,
+                0.03724697494506836,
+                0.03728384017944336,
+                0.037617664337158206,
+                0.03752140808105469,
+                0.03743641662597656,
+                0.03749068832397461,
+                0.037367809295654295,
+                0.037400577545166014,
+                0.037397502899169925,
+                0.03746713638305664,
+                0.03742534255981445,
+                0.03741062545776367,
+                0.03749068832397461,
+                0.037596160888671876,
+                0.037459968566894535,
+                0.03747635269165039,
+                0.037564414978027344,
+                0.03736166381835938,
+                0.03739043045043945,
+                0.03738000106811523,
+                0.037395454406738284,
+                0.037364734649658206,
+                0.037288959503173826,
+                0.037272575378417966,
+                0.03739033508300781,
+                0.03733606338500976,
+                0.03735756683349609,
+                0.03748761749267578,
+                0.03722956848144531,
+                0.03721011352539062,
+                0.03726438522338867,
+                0.03727974319458008,
+                0.0373054084777832,
+                0.0374005126953125,
+                0.03746406555175781,
+                0.03726335906982422,
+                0.03741388702392578,
+                0.03743027114868164,
+                0.037369857788085936,
+                0.037233665466308595,
+                0.03745075225830078,
+                0.03740364837646484,
+                0.03760537719726562,
+                0.0379504623413086,
+                0.037580799102783204,
+                0.037479423522949216,
+                0.03734527969360352,
+                0.037501953125,
+                0.03740364837646484,
+                0.03745382308959961,
+                0.03739136123657227,
+                0.03723161697387695,
+                0.03736678314208984,
+                0.03734425735473633,
+                0.037338111877441404,
+                0.0372490234375,
+                0.0378152961730957,
+                0.037308414459228514,
+                0.03731660842895508,
+                0.03753472137451172,
+                0.03752243041992188,
+                0.03744563293457031,
+                0.03733606338500976,
+                0.03758182525634766,
+                0.03740678405761719,
+                0.0373902702331543,
+                0.03753696060180664,
+                0.0376297607421875,
+                0.037501953125,
+                0.0375470085144043,
+                0.037407745361328126,
+                0.037338111877441404,
+                0.03747635269165039,
+                0.03741798400878906,
+                0.03750912094116211,
+                0.037425151824951174,
+                0.037394432067871096,
+                0.037370880126953124,
+                0.03738828659057617,
+                0.03724800109863281,
+                0.037305343627929685,
+                0.03722137451171875,
+                0.03721932983398438,
+                0.03724697494506836,
+                0.03733401489257813,
+                0.037212158203125,
+                0.037305343627929685,
+                0.03725721740722656,
+                0.03727462387084961,
+                0.03721932983398438,
+                0.037343456268310544,
+                0.03743619155883789,
+                0.037384193420410154,
+                0.0374466552734375,
+                0.037400577545166014,
+                0.03728486251831055,
+                0.03743948745727539,
+                0.0380579833984375,
+                0.03735551834106445,
+                0.03746815872192383,
+                0.037369857788085936,
+                0.03732787322998047,
+                0.03745587158203125,
+                0.037305343627929685,
+                0.03747532653808594,
+                0.03728384017944336,
+                0.03757567977905273,
+                0.03751424026489258,
+                0.037526527404785154,
+                0.0377149429321289,
+                0.0378603515625,
+                0.037510143280029294,
+                0.03727974319458008,
+                0.037449726104736326,
+                0.037510143280029294,
+                0.03718143844604492,
+                0.037466110229492186,
+                0.03742105484008789,
+                0.03767398452758789,
+                0.03747532653808594,
+                0.037292030334472655,
+                0.037585918426513674,
+                0.037443584442138675,
+                0.037375999450683595,
+                0.03736678314208984,
+                0.037539840698242184,
+                0.037429344177246096,
+                0.037510047912597655,
+                0.03733196640014649,
+                0.03733401489257813,
+                0.03750809478759766,
+                0.03741798400878906,
+                0.037558273315429686,
+                0.0375838737487793,
+                0.03759513473510742,
+                0.03748659133911133,
+                0.03753062438964844,
+                0.03750912094116211,
+                0.037561344146728515,
+                0.03765248107910156,
+                0.03770163345336914,
+                0.037580799102783204,
+                0.0376627197265625,
+                0.03765862274169922,
+                0.037694465637207034,
+                0.0375838737487793,
+                0.03765760040283203,
+                0.03766579055786133,
+                0.03772415924072266,
+                0.03762995147705078,
+                0.03750297546386719,
+                0.03774771118164062,
+                0.03756748962402344,
+                0.03758694458007812,
+                0.037539840698242184,
+                0.0373831672668457
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 26.72474704639313
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}