Upload /Llama-2-7b-hf-GPTQ-3bit/gptq3bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Llama-2-7b-hf-GPTQ-3bit/gptq3bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,603 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 3270.045696,
+            "max_vram": 4661.968896,
+            "max_reserved": 4177.526784,
+            "max_allocated": 3774.401024
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.357684063089305,
+            "stdev": 0.0008016350107244357,
+            "values": [
+                0.361810791015625,
+                0.357791748046875,
+                0.3575265197753906,
+                0.3581931457519531,
+                0.3574732666015625,
+                0.35754803466796875,
+                0.3574027099609375,
+                0.35748556518554686,
+                0.35731149291992187,
+                0.35762176513671873,
+                0.357396484375,
+                0.3575787658691406,
+                0.35739752197265623,
+                0.35757669067382813,
+                0.3574109802246094,
+                0.35748760986328126,
+                0.35739239501953124,
+                0.3574732666015625,
+                0.35765963745117185,
+                0.35760772705078125,
+                0.35759820556640626,
+                0.357516357421875,
+                0.35730328369140624,
+                0.35794638061523437,
+                0.3574128723144531,
+                0.3574732666015625,
+                0.35734527587890624,
+                0.35767193603515623,
+                0.35742413330078127
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 715.7154215620812
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 3270.045696,
+            "max_vram": 4771.0208,
+            "max_reserved": 4286.578688,
+            "max_allocated": 3941.391872
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 8.29499959564209,
+            "stdev": 0.22648267173766978,
+            "values": [
+                8.068516923904419,
+                8.521482267379758
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 30.74141198680327
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.03252941017898859,
+            "stdev": 0.0014214099317253592,
+            "values": [
+                0.03172454452514648,
+                0.031645696640014646,
+                0.03129964828491211,
+                0.03147052764892578,
+                0.0313436164855957,
+                0.031531007766723636,
+                0.031422464370727536,
+                0.03157708740234375,
+                0.03133030319213867,
+                0.03157708740234375,
+                0.031229951858520507,
+                0.03157017517089844,
+                0.03131599998474121,
+                0.03155427169799805,
+                0.03158323287963867,
+                0.031719423294067385,
+                0.03138457679748535,
+                0.03154044723510742,
+                0.031368064880371097,
+                0.03156163215637207,
+                0.031340543746948245,
+                0.03158527946472168,
+                0.031474687576293944,
+                0.03160985565185547,
+                0.03136211204528808,
+                0.031534015655517576,
+                0.03135286331176758,
+                0.03164566421508789,
+                0.03136000061035156,
+                0.031547391891479495,
+                0.03230515289306641,
+                0.031671327590942384,
+                0.031354015350341796,
+                0.0315984001159668,
+                0.031318016052246093,
+                0.03153526306152344,
+                0.03144278335571289,
+                0.03158835220336914,
+                0.031559680938720705,
+                0.0315996150970459,
+                0.03142758369445801,
+                0.031524864196777344,
+                0.03141939163208008,
+                0.031633440017700194,
+                0.03130467224121094,
+                0.03155763244628906,
+                0.031558656692504884,
+                0.031729856491088865,
+                0.03134239959716797,
+                0.031491071701049804,
+                0.03135696029663086,
+                0.03149001693725586,
+                0.031335424423217774,
+                0.03153715133666992,
+                0.03148287963867188,
+                0.031633695602416995,
+                0.03132499122619629,
+                0.03150531196594238,
+                0.031347776412963865,
+                0.03160569572448731,
+                0.03131699180603027,
+                0.03157606315612793,
+                0.03143680000305176,
+                0.03154431915283203,
+                0.03135897636413574,
+                0.03149619293212891,
+                0.031308799743652346,
+                0.03142451286315918,
+                0.03134582328796387,
+                0.03138748741149902,
+                0.03200831985473633,
+                0.031571840286254885,
+                0.03134668731689453,
+                0.03140505599975586,
+                0.031337472915649416,
+                0.03144396781921387,
+                0.03131699180603027,
+                0.03136204719543457,
+                0.031486976623535154,
+                0.03149619293212891,
+                0.031226879119873048,
+                0.0315003833770752,
+                0.031268768310546875,
+                0.03129548835754394,
+                0.03127603149414063,
+                0.03143168067932129,
+                0.03143270492553711,
+                0.031488000869750975,
+                0.031297536849975584,
+                0.03136204719543457,
+                0.03120947265625,
+                0.0314019832611084,
+                0.031264768600463864,
+                0.03136511993408203,
+                0.03146956825256347,
+                0.0314204158782959,
+                0.03128118324279785,
+                0.031433984756469725,
+                0.0312706241607666,
+                0.03138764762878418,
+                0.031309951782226564,
+                0.031315839767456055,
+                0.03203593444824219,
+                0.03144694328308106,
+                0.0313702392578125,
+                0.03136000061035156,
+                0.03134259223937988,
+                0.03144704055786133,
+                0.03239731216430664,
+                0.031509504318237305,
+                0.03160166358947754,
+                0.03150140762329102,
+                0.031220640182495117,
+                0.031460351943969726,
+                0.031437824249267575,
+                0.03141529655456543,
+                0.031336448669433595,
+                0.03146240043640137,
+                0.031491071701049804,
+                0.03148185539245606,
+                0.031336448669433595,
+                0.0314726390838623,
+                0.03139798355102539,
+                0.03150428771972656,
+                0.031303680419921875,
+                0.03151769638061523,
+                0.031399168014526364,
+                0.03144166374206543,
+                0.03153817558288574,
+                0.03167948722839355,
+                0.031323135375976564,
+                0.031510528564453126,
+                0.03133440017700195,
+                0.03138470458984375,
+                0.031677600860595706,
+                0.0315328311920166,
+                0.03125964736938477,
+                0.0314080638885498,
+                0.0313436164855957,
+                0.031488000869750975,
+                0.03134873580932617,
+                0.03139583969116211,
+                0.03196928024291992,
+                0.031455232620239255,
+                0.031221759796142577,
+                0.03141427230834961,
+                0.03127609634399414,
+                0.031447999954223635,
+                0.031506624221801754,
+                0.03154924774169922,
+                0.03132928085327148,
+                0.031440895080566404,
+                0.031301631927490234,
+                0.03136000061035156,
+                0.03129651260375976,
+                0.03136511993408203,
+                0.03124224090576172,
+                0.03149004745483398,
+                0.03128934478759766,
+                0.031486976623535154,
+                0.03129958343505859,
+                0.031336448669433595,
+                0.031181087493896486,
+                0.031441823959350586,
+                0.03129631996154785,
+                0.03147980880737305,
+                0.03140505599975586,
+                0.031512575149536134,
+                0.031357952117919925,
+                0.03144806480407715,
+                0.03133657646179199,
+                0.03141526412963867,
+                0.031632383346557616,
+                0.031696992874145506,
+                0.03150956726074219,
+                0.031551231384277345,
+                0.031301631927490234,
+                0.03139686393737793,
+                0.031291391372680666,
+                0.031388671875,
+                0.031244415283203125,
+                0.03207276916503906,
+                0.031389503479003905,
+                0.031470592498779294,
+                0.03123200035095215,
+                0.0314071044921875,
+                0.031291391372680666,
+                0.03134259223937988,
+                0.03128319931030273,
+                0.03150870323181152,
+                0.03139267158508301,
+                0.03240127944946289,
+                0.032894977569580076,
+                0.032917537689208985,
+                0.03270755386352539,
+                0.03164159965515137,
+                0.03151872062683105,
+                0.0315361270904541,
+                0.0314521598815918,
+                0.03160371208190918,
+                0.03187302398681641,
+                0.03152105522155762,
+                0.03127881622314453,
+                0.031410175323486327,
+                0.03134259223937988,
+                0.031680511474609374,
+                0.03155353546142578,
+                0.03149932861328125,
+                0.03287545776367187,
+                0.03464704132080078,
+                0.03443215942382812,
+                0.034573150634765626,
+                0.034365726470947267,
+                0.033939361572265625,
+                0.034250560760498046,
+                0.0344268798828125,
+                0.03438694381713867,
+                0.0345354232788086,
+                0.034210079193115236,
+                0.034248416900634765,
+                0.0321710090637207,
+                0.031587488174438474,
+                0.03144483184814453,
+                0.031566015243530275,
+                0.03135980796813965,
+                0.031415519714355467,
+                0.03133622360229492,
+                0.031373311996459964,
+                0.03146956825256347,
+                0.03145948791503906,
+                0.03131171226501465,
+                0.0314654712677002,
+                0.03167231941223145,
+                0.03163340759277344,
+                0.031319040298461914,
+                0.03146342468261719,
+                0.03157196807861328,
+                0.031632383346557616,
+                0.03151155281066895,
+                0.031480831146240236,
+                0.031318016052246093,
+                0.03142348861694336,
+                0.03135385513305664,
+                0.0314337272644043,
+                0.0314768009185791,
+                0.03153327941894531,
+                0.03134639930725098,
+                0.03143075180053711,
+                0.03140188789367676,
+                0.03140003204345703,
+                0.03128105545043945,
+                0.03142758369445801,
+                0.0341739501953125,
+                0.034342910766601564,
+                0.03410438537597656,
+                0.03159449577331543,
+                0.0350382080078125,
+                0.034541568756103515,
+                0.03455692672729492,
+                0.03449971389770508,
+                0.03449740982055664,
+                0.03425791931152344,
+                0.03443199920654297,
+                0.03409408187866211,
+                0.0341923828125,
+                0.034446334838867186,
+                0.034409759521484375,
+                0.03426889419555664,
+                0.0344719352722168,
+                0.0341739501953125,
+                0.03430809783935547,
+                0.034108417510986325,
+                0.03447619247436524,
+                0.0342507209777832,
+                0.03149401664733887,
+                0.03132928085327148,
+                0.03135385513305664,
+                0.031264768600463864,
+                0.03345209503173828,
+                0.03460192108154297,
+                0.03453852844238281,
+                0.03425491333007812,
+                0.03441347122192383,
+                0.034167808532714845,
+                0.034331649780273435,
+                0.03390476989746094,
+                0.03420249557495117,
+                0.03420979309082031,
+                0.034164798736572265,
+                0.03412473678588867,
+                0.034260990142822266,
+                0.033990657806396485,
+                0.03421184158325195,
+                0.033584129333496096,
+                0.03415654373168946,
+                0.03424256134033203,
+                0.034359294891357424,
+                0.03419955062866211,
+                0.03431939315795898,
+                0.03419644927978516,
+                0.040233985900878906,
+                0.033919998168945316,
+                0.03439513778686523,
+                0.03411049652099609,
+                0.03444425582885742,
+                0.03434905624389648,
+                0.03437055969238281,
+                0.03479244613647461,
+                0.034353214263916014,
+                0.03416979217529297,
+                0.03433273696899414,
+                0.0342465934753418,
+                0.03429785537719727,
+                0.03422208023071289,
+                0.034290687561035156,
+                0.03410227203369141,
+                0.034348033905029295,
+                0.0343111686706543,
+                0.03423859024047852,
+                0.03426607894897461,
+                0.0344136962890625,
+                0.034316062927246094,
+                0.033805313110351565,
+                0.03429683303833008,
+                0.03424051284790039,
+                0.03429683303833008,
+                0.0342927360534668,
+                0.034170879364013675,
+                0.03436649703979492,
+                0.03407664108276367,
+                0.03441459274291992,
+                0.03440947341918945,
+                0.03445452880859375,
+                0.034367488861083983,
+                0.03446492767333984,
+                0.03435820770263672,
+                0.0342476806640625,
+                0.03398031997680664,
+                0.03432755279541016,
+                0.034167808532714845,
+                0.03439206314086914,
+                0.03435417556762695,
+                0.034408447265625,
+                0.03491839981079101,
+                0.03421286392211914,
+                0.03435327911376953,
+                0.03425369644165039,
+                0.034236415863037106,
+                0.034375679016113284,
+                0.03423846435546875,
+                0.03433267211914062,
+                0.03385139083862305,
+                0.0341319694519043,
+                0.03419647979736328,
+                0.03427020645141601,
+                0.034304000854492187,
+                0.03440435028076172,
+                0.03424563217163086,
+                0.034028545379638675,
+                0.03394559860229492,
+                0.03431423950195313,
+                0.03422832107543945,
+                0.03436636734008789,
+                0.032497665405273435,
+                0.03160985565185547,
+                0.031214591979980468,
+                0.03416166305541992,
+                0.03431935882568359,
+                0.03447808074951172,
+                0.0343111686706543,
+                0.03432156753540039,
+                0.03433977508544922,
+                0.034485153198242184,
+                0.034628608703613284,
+                0.03445065689086914,
+                0.034345760345458984,
+                0.034356224060058595,
+                0.034304000854492187,
+                0.03437977600097656,
+                0.03428764724731445,
+                0.034246623992919924,
+                0.03394867324829102,
+                0.0344002571105957,
+                0.03459913635253906,
+                0.03461404800415039,
+                0.034338081359863284,
+                0.03449212646484375,
+                0.034280670166015624,
+                0.03393718338012695,
+                0.03432550430297852,
+                0.0344002571105957,
+                0.034283519744873044,
+                0.03434905624389648,
+                0.034487518310546875,
+                0.034417633056640626,
+                0.03381433486938477,
+                0.0342848014831543,
+                0.034323200225830075,
+                0.03427840042114258,
+                0.03425689697265625,
+                0.03449039840698242,
+                0.034312286376953126,
+                0.034184062957763675,
+                0.03167436790466309,
+                0.03170918464660644,
+                0.03140300750732422,
+                0.03154556846618652,
+                0.03140687942504883,
+                0.03150131225585937,
+                0.03133132743835449,
+                0.03219660949707031,
+                0.03135897636413574,
+                0.03145011138916016,
+                0.03128838348388672,
+                0.03133638381958008,
+                0.031319232940673826,
+                0.03131987190246582,
+                0.03133760070800781,
+                0.03154431915283203,
+                0.031265663146972655,
+                0.03135910415649414,
+                0.03125235176086426,
+                0.03138662338256836,
+                0.031238143920898437,
+                0.03137228775024414,
+                0.031244287490844725,
+                0.0314839038848877,
+                0.03162931251525879,
+                0.031643648147583005,
+                0.03128748893737793,
+                0.0315216007232666,
+                0.03124940872192383,
+                0.031406143188476565,
+                0.03131180763244629,
+                0.031492095947265625,
+                0.0313089599609375,
+                0.031408992767333985,
+                0.031244287490844725,
+                0.031356096267700195,
+                0.031178560256958008,
+                0.031380544662475585,
+                0.03124835205078125,
+                0.031534048080444334,
+                0.031320064544677735,
+                0.031341567993164066,
+                0.031245567321777343,
+                0.03142937660217285,
+                0.031251455307006834,
+                0.03138355255126953,
+                0.031304704666137696,
+                0.03416371154785156,
+                0.03425177764892578,
+                0.03444035339355469,
+                0.034200416564941404,
+                0.0344453125,
+                0.0316693115234375,
+                0.03159647941589355,
+                0.03226009750366211,
+                0.03422822570800781,
+                0.03422412872314453,
+                0.03432550430297852,
+                0.034290687561035156,
+                0.034513919830322266,
+                0.03430912017822266,
+                0.03448524856567383,
+                0.03404800033569336,
+                0.03444326400756836,
+                0.03440332794189453,
+                0.03445657730102539,
+                0.03427532958984375,
+                0.0344637451171875,
+                0.03435724639892578,
+                0.034157569885253904,
+                0.03413423919677734,
+                0.03447596740722656,
+                0.03439702224731445,
+                0.03454771041870117,
+                0.03423436737060547,
+                0.03422515106201172,
+                0.03398147201538086,
+                0.034051166534423825,
+                0.034279296875,
+                0.034398368835449215,
+                0.03441136169433594,
+                0.034476032257080076,
+                0.03429683303833008,
+                0.03484697723388672,
+                0.03185433578491211,
+                0.03169587135314941,
+                0.03143065643310547,
+                0.03160780715942383,
+                0.031476736068725586,
+                0.031604736328125,
+                0.03144707107543945,
+                0.031568864822387695,
+                0.03142451286315918,
+                0.03157097625732422,
+                0.031301824569702145,
+                0.031432479858398435,
+                0.03136409568786621,
+                0.03146751976013184,
+                0.03139379119873047,
+                0.03154944038391113,
+                0.03139379119873047,
+                0.031474687576293944,
+                0.031301631927490234,
+                0.03138355255126953,
+                0.03134873580932617,
+                0.031410175323486327,
+                0.03127807998657227
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 30.741411986803264
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}