Upload /Llama-2-13b-hf/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Llama-2-13b-hf/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,438 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1019.068416,
+            "max_vram": 8359.247872,
+            "max_reserved": 7874.80576,
+            "max_allocated": 7776.413184
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.08362868264516198,
+            "stdev": 0.0005853808320203658,
+            "values": [
+                0.08983631896972656,
+                0.08470630645751953,
+                0.08380518341064454,
+                0.08375091552734375,
+                0.08335871887207032,
+                0.08358297729492188,
+                0.08349183654785156,
+                0.08354611206054688,
+                0.0835225601196289,
+                0.08355020904541016,
+                0.08349491119384765,
+                0.08333312225341796,
+                0.08380108642578125,
+                0.08340889739990234,
+                0.0835225601196289,
+                0.08359117126464843,
+                0.08348876953125,
+                0.08336589050292968,
+                0.08354508972167969,
+                0.08330137634277343,
+                0.08355840301513671,
+                0.08343961334228515,
+                0.08356352233886719,
+                0.08346419525146484,
+                0.08360652923583985,
+                0.08351846313476563,
+                0.08354713439941407,
+                0.08336691284179687,
+                0.08351538848876953,
+                0.08338534545898438,
+                0.08351334381103516,
+                0.08355328369140624,
+                0.08356771087646485,
+                0.08354918670654297,
+                0.08356761932373047,
+                0.0835420150756836,
+                0.08347238159179687,
+                0.08361779022216796,
+                0.0835225601196289,
+                0.08360243225097656,
+                0.08349491119384765,
+                0.0835962905883789,
+                0.08337100982666015,
+                0.08359219360351562,
+                0.08346931457519531,
+                0.08357990264892579,
+                0.08357785797119141,
+                0.08359731292724609,
+                0.0834897918701172,
+                0.08348569488525391,
+                0.08356658935546875,
+                0.08365465545654296,
+                0.08348774719238282,
+                0.08371814727783203,
+                0.08349798583984375,
+                0.0836485137939453,
+                0.08347545623779297,
+                0.08364543914794922,
+                0.08351232147216797,
+                0.08361478424072266,
+                0.0835594253540039,
+                0.08360447692871094,
+                0.08349491119384765,
+                0.0835389404296875,
+                0.08356147003173828,
+                0.08361984252929687,
+                0.08354713439941407,
+                0.08363008117675781,
+                0.08351846313476563,
+                0.08367411041259766,
+                0.08348473358154297,
+                0.0836659164428711,
+                0.0835440673828125,
+                0.0836864013671875,
+                0.08336399841308594,
+                0.08362393951416015,
+                0.08365670776367187,
+                0.08365670776367187,
+                0.08352767944335937,
+                0.08361676788330077,
+                0.08363314819335937,
+                0.08359423828125,
+                0.08361164855957032,
+                0.08357068634033203,
+                0.08360755157470703,
+                0.08362086486816406,
+                0.08364031982421875,
+                0.083631103515625,
+                0.0836648941040039,
+                0.08355532836914062,
+                0.08365567779541015,
+                0.08343654632568359,
+                0.08357785797119141,
+                0.08364339447021485,
+                0.08360249328613281,
+                0.08353485107421875,
+                0.08357990264892579,
+                0.08367616271972657,
+                0.08360345458984375,
+                0.08363212585449219,
+                0.08361062622070313,
+                0.08357785797119141,
+                0.08362290954589843,
+                0.08356864166259766,
+                0.08365670776367187,
+                0.08355532836914062,
+                0.08361881256103515,
+                0.08357273864746094,
+                0.08354918670654297,
+                0.08361472320556641,
+                0.0835758056640625,
+                0.0836157455444336,
+                0.083736572265625,
+                0.08359935760498047,
+                0.08367001342773438,
+                0.08368434906005859,
+                0.08361676788330077,
+                0.08370073699951172,
+                0.08360857391357422,
+                0.08348467254638672
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 3061.1506949859854
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1019.068416,
+            "max_vram": 9105.833984,
+            "max_reserved": 8621.391872,
+            "max_allocated": 8484.93824
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 10.647108638763433,
+            "stdev": 0,
+            "values": [
+                10.647108638763433
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 23.950164185571417
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.04175336721083699,
+            "stdev": 0.00015897738186358224,
+            "values": [
+                0.04219801712036133,
+                0.04244070434570312,
+                0.04166758346557617,
+                0.04163174438476563,
+                0.04178432083129883,
+                0.04164947128295898,
+                0.041617088317871094,
+                0.04190729522705078,
+                0.041731998443603514,
+                0.0417097282409668,
+                0.04163772964477539,
+                0.041665534973144534,
+                0.04169113540649414,
+                0.04170444869995117,
+                0.041525344848632816,
+                0.041611392974853514,
+                0.041853729248046874,
+                0.041670654296875,
+                0.04159078216552734,
+                0.0422031364440918,
+                0.04166348648071289,
+                0.04166144180297852,
+                0.04178636932373047,
+                0.04178227233886719,
+                0.04161740875244141,
+                0.041637889862060545,
+                0.041540607452392575,
+                0.041680896759033206,
+                0.04149350357055664,
+                0.041853950500488284,
+                0.04163891220092773,
+                0.042157054901123044,
+                0.041724929809570314,
+                0.04184678268432617,
+                0.04176998519897461,
+                0.04156313705444336,
+                0.041793537139892575,
+                0.04184064102172851,
+                0.041731071472167966,
+                0.04159795379638672,
+                0.041635841369628904,
+                0.041799678802490234,
+                0.04178841781616211,
+                0.04166860961914062,
+                0.04155392074584961,
+                0.04159795379638672,
+                0.04189081573486328,
+                0.04163593673706055,
+                0.04162140655517578,
+                0.041591808319091796,
+                0.04174848175048828,
+                0.04168505477905273,
+                0.04173100662231445,
+                0.04184883117675781,
+                0.04174335861206055,
+                0.04178636932373047,
+                0.04176998519897461,
+                0.04193280029296875,
+                0.04166451263427735,
+                0.04170649719238281,
+                0.041626625061035157,
+                0.04183552169799805,
+                0.0415549430847168,
+                0.04173619079589844,
+                0.04194303894042969,
+                0.04162876892089844,
+                0.041638816833496094,
+                0.04215091323852539,
+                0.041744384765625,
+                0.0415428466796875,
+                0.04168377685546875,
+                0.04152937698364258,
+                0.04166140747070313,
+                0.0417259521484375,
+                0.04163993453979492,
+                0.04149248123168945,
+                0.041828353881835936,
+                0.04155801773071289,
+                0.04156825637817383,
+                0.04218163299560547,
+                0.04151910400390625,
+                0.04177519989013672,
+                0.041640865325927735,
+                0.04164198303222656,
+                0.04178841781616211,
+                0.04186127853393555,
+                0.041912158966064456,
+                0.04161228942871094,
+                0.04188774490356445,
+                0.041880577087402344,
+                0.04161228942871094,
+                0.04169126510620117,
+                0.041487232208251956,
+                0.04148857498168945,
+                0.04159161758422852,
+                0.04171980667114258,
+                0.04153855895996094,
+                0.04185190582275391,
+                0.04175769424438477,
+                0.04186223983764648,
+                0.04184156799316406,
+                0.04189801788330078,
+                0.04187030410766602,
+                0.041836544036865236,
+                0.041780223846435545,
+                0.041708545684814455,
+                0.04182425689697265,
+                0.04182947158813476,
+                0.04204636764526367,
+                0.041854209899902343,
+                0.041729793548583985,
+                0.04175974273681641,
+                0.041829376220703124,
+                0.041714687347412106,
+                0.04189286422729492,
+                0.04172390365600586,
+                0.04212428665161133,
+                0.041708545684814455,
+                0.041812992095947264,
+                0.041919486999511715,
+                0.04166860961914062,
+                0.04163174438476563,
+                0.04183859252929688,
+                0.04176793670654297,
+                0.041902080535888675,
+                0.04172902297973633,
+                0.04196352005004883,
+                0.04229324722290039,
+                0.041799678802490234,
+                0.04192153549194336,
+                0.0418682861328125,
+                0.0418600959777832,
+                0.04173516845703125,
+                0.041987071990966796,
+                0.04173926544189453,
+                0.04172288131713867,
+                0.04189286422729492,
+                0.041780223846435545,
+                0.041670654296875,
+                0.04171673583984375,
+                0.041902080535888675,
+                0.04194713592529297,
+                0.04179455947875976,
+                0.04173926544189453,
+                0.04173516845703125,
+                0.04178636932373047,
+                0.04187238311767578,
+                0.041665534973144534,
+                0.04189286422729492,
+                0.04149760055541992,
+                0.04184473419189453,
+                0.04191436767578125,
+                0.04183244705200195,
+                0.042280223846435545,
+                0.042299102783203125,
+                0.04186214447021484,
+                0.04178841781616211,
+                0.04185497665405274,
+                0.04175769424438477,
+                0.04167782211303711,
+                0.04173516845703125,
+                0.041816062927246093,
+                0.04177203369140625,
+                0.0417894401550293,
+                0.042105857849121096,
+                0.04157654571533203,
+                0.04186614227294922,
+                0.041842689514160154,
+                0.041605121612548826,
+                0.04171878433227539,
+                0.041629695892333986,
+                0.041875457763671874,
+                0.04161536026000977,
+                0.04171366500854492,
+                0.04176588821411133,
+                0.04219596862792969,
+                0.04166144180297852,
+                0.04165836715698242,
+                0.041659393310546876,
+                0.04185625457763672,
+                0.04177382278442383,
+                0.04182732772827148,
+                0.041847808837890625,
+                0.041826305389404295,
+                0.04172083282470703,
+                0.04182527923583984,
+                0.0418600959777832,
+                0.04174950408935547,
+                0.04171571350097656,
+                0.041708545684814455,
+                0.04163686370849609,
+                0.04158464050292969,
+                0.041665534973144534,
+                0.04190719985961914,
+                0.041680896759033206,
+                0.041777153015136716,
+                0.041534465789794923,
+                0.04157136154174805,
+                0.04159382247924805,
+                0.04162355041503906,
+                0.04171571350097656,
+                0.041673728942871094,
+                0.041724929809570314,
+                0.041665534973144534,
+                0.041594879150390625,
+                0.04173209762573242,
+                0.04168294525146484,
+                0.04171980667114258,
+                0.0416890869140625,
+                0.04168601608276367,
+                0.041556991577148435,
+                0.041712928771972656,
+                0.042162910461425784,
+                0.04135424041748047,
+                0.04165119934082031,
+                0.04175872039794922,
+                0.04175564956665039,
+                0.041715999603271485,
+                0.041715423583984376,
+                0.041567230224609376,
+                0.04177305603027344,
+                0.04170348739624023,
+                0.041805759429931644,
+                0.041662464141845705,
+                0.04213555145263672,
+                0.04206489562988281,
+                0.04169113540649414,
+                0.04160531234741211,
+                0.041748287200927735,
+                0.04173516845703125,
+                0.041632766723632815,
+                0.04172902297973633,
+                0.04197683334350586,
+                0.041750526428222655,
+                0.041635841369628904,
+                0.04169318389892578,
+                0.04157747268676758,
+                0.041750526428222655,
+                0.04165017700195312,
+                0.04164198303222656,
+                0.04153855895996094,
+                0.0416184310913086,
+                0.04174540710449219,
+                0.041657344818115234,
+                0.041752574920654296,
+                0.04167782211303711,
+                0.041971710205078124,
+                0.04169625473022461,
+                0.04170342254638672,
+                0.04166041564941406,
+                0.041545726776123046,
+                0.04167270278930664,
+                0.04174335861206055,
+                0.04185190582275391,
+                0.041777153015136716
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 23.950164185571413
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}