Upload /Llama-2-13b-hf/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Llama-2-13b-hf/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,438 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1040.412672,
+            "max_vram": 8359.247872,
+            "max_reserved": 7874.80576,
+            "max_allocated": 7776.413184
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.08359146785736085,
+            "stdev": 0.0003168160271875822,
+            "values": [
+                0.08683526611328125,
+                0.08383897399902343,
+                0.08395785522460937,
+                0.08395468902587891,
+                0.08392806243896485,
+                0.08374681854248046,
+                0.08348365020751954,
+                0.08350617980957031,
+                0.08352767944335937,
+                0.0835389404296875,
+                0.08349388885498046,
+                0.08330547332763671,
+                0.08352476501464844,
+                0.08337407684326172,
+                0.08358809661865234,
+                0.08357683563232422,
+                0.08349696350097656,
+                0.08338636779785157,
+                0.08351757049560547,
+                0.08328102111816406,
+                0.083557373046875,
+                0.0834150390625,
+                0.08355328369140624,
+                0.0834897918701172,
+                0.08355840301513671,
+                0.08347241973876954,
+                0.08351948547363282,
+                0.08340275573730468,
+                0.08354918670654297,
+                0.08337612915039062,
+                0.08370687866210938,
+                0.08353177642822265,
+                0.08353075408935547,
+                0.08354415893554687,
+                0.08349286651611328,
+                0.08354303741455078,
+                0.08346112060546874,
+                0.08360140991210938,
+                0.083525634765625,
+                0.0836157455444336,
+                0.08351948547363282,
+                0.08360652923583985,
+                0.08336486053466798,
+                0.08361369323730469,
+                0.0834150390625,
+                0.08355430603027343,
+                0.08352767944335937,
+                0.08360243225097656,
+                0.08346214294433593,
+                0.08346623992919922,
+                0.08349900817871093,
+                0.08363212585449219,
+                0.08352051544189452,
+                0.08367513275146485,
+                0.0834672622680664,
+                0.08364543914794922,
+                0.08342937469482421,
+                0.08361984252929687,
+                0.08352460479736327,
+                0.08361062622070313,
+                0.08352665710449218,
+                0.08359219360351562,
+                0.08350208282470703,
+                0.08358624267578126,
+                0.08351129913330078,
+                0.08362290954589843,
+                0.08356147003173828,
+                0.08361369323730469,
+                0.08354611206054688,
+                0.08366899108886719,
+                0.08349491119384765,
+                0.08363827514648438,
+                0.08353997039794922,
+                0.08369356536865234,
+                0.083346435546875,
+                0.0835962905883789,
+                0.08361369323730469,
+                0.08360652923583985,
+                0.08347955322265625,
+                0.08356249237060547,
+                0.0835758056640625,
+                0.08359219360351562,
+                0.0835962905883789,
+                0.08353587341308594,
+                0.08360870361328125,
+                0.083525634765625,
+                0.08362188720703125,
+                0.08359219360351562,
+                0.08364031982421875,
+                0.08353791809082031,
+                0.08360140991210938,
+                0.08339353942871094,
+                0.08354713439941407,
+                0.08360857391357422,
+                0.08356454467773437,
+                0.083557373046875,
+                0.08356864166259766,
+                0.08348377227783203,
+                0.08361676788330077,
+                0.08360038757324219,
+                0.08362188720703125,
+                0.08355123138427735,
+                0.08360550689697266,
+                0.08351948547363282,
+                0.08361369323730469,
+                0.08356966400146484,
+                0.08360959625244141,
+                0.08357478332519531,
+                0.08351948547363282,
+                0.08359526062011718,
+                0.08355328369140624,
+                0.08358399963378907,
+                0.08374169921875,
+                0.08360959625244141,
+                0.08370585632324219,
+                0.08384614562988281,
+                0.08362700653076172,
+                0.08367820739746094,
+                0.08360550689697266,
+                0.08350514984130859
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 3062.5135143796533
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1040.412672,
+            "max_vram": 9105.833984,
+            "max_reserved": 8621.391872,
+            "max_allocated": 8484.93824
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 10.627742668151864,
+            "stdev": 0,
+            "values": [
+                10.627742668151864
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 23.993806395421863
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.041677422228046525,
+            "stdev": 0.0012397732588279448,
+            "values": [
+                0.04167164611816406,
+                0.042102783203125,
+                0.0420055046081543,
+                0.04145151901245117,
+                0.041191425323486325,
+                0.04131737518310547,
+                0.04123136138916016,
+                0.04142710494995117,
+                0.04152918243408203,
+                0.041420799255371094,
+                0.041488384246826174,
+                0.041543678283691404,
+                0.04138905715942383,
+                0.041818111419677735,
+                0.04131020736694336,
+                0.04139116668701172,
+                0.04164499282836914,
+                0.04136140823364258,
+                0.04150476837158203,
+                0.04135935974121094,
+                0.04147814559936523,
+                0.04135935974121094,
+                0.04153964614868164,
+                0.0413337287902832,
+                0.04142179107666016,
+                0.04132761764526367,
+                0.04142489624023438,
+                0.04138905715942383,
+                0.041338878631591795,
+                0.04162563323974609,
+                0.0413337287902832,
+                0.04134092712402344,
+                0.04127948760986328,
+                0.04148019027709961,
+                0.041409534454345705,
+                0.04124684906005859,
+                0.04147494506835937,
+                0.04141260910034179,
+                0.04144025421142578,
+                0.04137881469726563,
+                0.04145151901245117,
+                0.04148019027709961,
+                0.04151398468017578,
+                0.04123545455932617,
+                0.04150374221801758,
+                0.04131024169921875,
+                0.04148937606811524,
+                0.04153958511352539,
+                0.041419776916503906,
+                0.04144844818115234,
+                0.04202905654907227,
+                0.04156620788574219,
+                0.041452545166015625,
+                0.04154880142211914,
+                0.0415098876953125,
+                0.04137881469726563,
+                0.041576446533203124,
+                0.0413941764831543,
+                0.041322494506835936,
+                0.04136140823364258,
+                0.041411582946777346,
+                0.041336830139160154,
+                0.04154265594482422,
+                0.04140236663818359,
+                0.04135628890991211,
+                0.0413757438659668,
+                0.04136038589477539,
+                0.04141363143920898,
+                0.04146278381347656,
+                0.041279518127441406,
+                0.04157027053833008,
+                0.041524223327636715,
+                0.041562110900878906,
+                0.04135935974121094,
+                0.041441280364990236,
+                0.04146995162963867,
+                0.04131430435180664,
+                0.04137472152709961,
+                0.04149964904785156,
+                0.04145459365844727,
+                0.04140236663818359,
+                0.041495582580566404,
+                0.04150780868530273,
+                0.04149673461914063,
+                0.04136636734008789,
+                0.04160720062255859,
+                0.041773025512695315,
+                0.041404415130615234,
+                0.041474048614501956,
+                0.041565185546875,
+                0.04143820953369141,
+                0.041468929290771485,
+                0.041299968719482424,
+                0.041422847747802735,
+                0.04123955154418945,
+                0.04147711944580078,
+                0.04130508804321289,
+                0.04152041625976562,
+                0.04150140762329101,
+                0.04227993774414063,
+                0.04153958511352539,
+                0.04153855895996094,
+                0.041431041717529295,
+                0.04143308639526367,
+                0.04128255844116211,
+                0.04138086318969727,
+                0.041166847229003906,
+                0.041262081146240234,
+                0.041567230224609376,
+                0.041502975463867185,
+                0.04141337585449219,
+                0.041924606323242186,
+                0.041352191925048826,
+                0.041439231872558595,
+                0.04144025421142578,
+                0.041335807800292966,
+                0.04135424041748047,
+                0.041588737487792966,
+                0.04134707260131836,
+                0.04145459365844727,
+                0.04140851211547852,
+                0.041332736968994144,
+                0.041470977783203126,
+                0.04137779235839844,
+                0.04135833740234375,
+                0.0412149772644043,
+                0.04135321426391601,
+                0.04133478546142578,
+                0.04139212799072266,
+                0.04152524948120117,
+                0.04153855895996094,
+                0.04152323150634766,
+                0.041465824127197265,
+                0.04138598251342773,
+                0.041596927642822266,
+                0.04160921478271484,
+                0.04134195327758789,
+                0.04149760055541992,
+                0.041506816864013675,
+                0.041171966552734376,
+                0.041376766204833985,
+                0.04130303955078125,
+                0.041485313415527345,
+                0.04133171081542969,
+                0.04147609710693359,
+                0.04129587173461914,
+                0.04152115249633789,
+                0.04149769592285156,
+                0.041712543487548825,
+                0.04134627151489258,
+                0.04153833770751953,
+                0.041565185546875,
+                0.041336830139160154,
+                0.04177305603027344,
+                0.04197273635864258,
+                0.04155295944213867,
+                0.041364414215087894,
+                0.041474048614501956,
+                0.04139212799072266,
+                0.042019840240478515,
+                0.041366527557373044,
+                0.04149760055541992,
+                0.04135116958618164,
+                0.04129075241088867,
+                0.04142489624023438,
+                0.04135833740234375,
+                0.04156927871704102,
+                0.04133785629272461,
+                0.04155084609985352,
+                0.046461952209472655,
+                0.04880588912963867,
+                0.04842803192138672,
+                0.04739276885986328,
+                0.04712140655517578,
+                0.04693503952026367,
+                0.04720025634765625,
+                0.04616191864013672,
+                0.043786239624023435,
+                0.04301311874389648,
+                0.045399040222167966,
+                0.050776065826416014,
+                0.043940864562988284,
+                0.041614334106445314,
+                0.041376766204833985,
+                0.041444351196289066,
+                0.04132352066040039,
+                0.04131737518310547,
+                0.041277599334716794,
+                0.04122812652587891,
+                0.041248767852783204,
+                0.0413829116821289,
+                0.04146585464477539,
+                0.04129075241088867,
+                0.04157440185546875,
+                0.04157132720947266,
+                0.04169625473022461,
+                0.04134297561645508,
+                0.04142899322509765,
+                0.04124671936035156,
+                0.04134707260131836,
+                0.041422847747802735,
+                0.0411412467956543,
+                0.0415365104675293,
+                0.04128684616088867,
+                0.04138374328613281,
+                0.04112691116333008,
+                0.04149248123168945,
+                0.04140851211547852,
+                0.04108595275878906,
+                0.041388031005859374,
+                0.04128255844116211,
+                0.04140057754516602,
+                0.041223934173583984,
+                0.04111974334716797,
+                0.04141363143920898,
+                0.04128460693359375,
+                0.04168806457519531,
+                0.041196544647216796,
+                0.041545726776123046,
+                0.04132147216796875,
+                0.04140236663818359,
+                0.0413040657043457,
+                0.041218048095703126,
+                0.04111667251586914,
+                0.04108902359008789,
+                0.04108595275878906,
+                0.04117708969116211,
+                0.04120064163208008,
+                0.04139622497558594,
+                0.04127449417114258,
+                0.04130086517333984,
+                0.04131532669067383,
+                0.04116070556640625,
+                0.04123852920532227,
+                0.04122623825073242,
+                0.041095169067382815,
+                0.04112384033203125,
+                0.04123555374145508,
+                0.041143199920654294,
+                0.04131532669067383,
+                0.04128768157958984,
+                0.041073665618896485,
+                0.041371646881103515,
+                0.04150271987915039,
+                0.041594879150390625,
+                0.04117913436889648,
+                0.04128255844116211,
+                0.041371646881103515,
+                0.04132761764526367,
+                0.04136345672607422,
+                0.041285633087158206,
+                0.04130815887451172,
+                0.04104806518554688,
+                0.041336830139160154,
+                0.04110745620727539
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 23.993806395421863
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}