Upload /Mistral-7B-v0.1-GPTQ-2bit/gptq2bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Mistral-7B-v0.1-GPTQ-2bit/gptq2bit_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,636 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 4967.591936,
+            "max_vram": 3894.411264,
+            "max_reserved": 3409.969152,
+            "max_allocated": 3288.941056
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.1639428302395728,
+            "stdev": 0.0009759040654938562,
+            "values": [
+                0.17022662353515625,
+                0.16785714721679687,
+                0.1638983612060547,
+                0.16406243896484374,
+                0.16345394897460938,
+                0.16390757751464843,
+                0.16359834289550781,
+                0.1638830108642578,
+                0.16355020141601562,
+                0.16390451049804688,
+                0.16350515747070313,
+                0.16400384521484376,
+                0.16355532836914063,
+                0.1639147491455078,
+                0.16354098510742188,
+                0.16405810546875,
+                0.16360652160644532,
+                0.16395791625976563,
+                0.1635635223388672,
+                0.16390963745117187,
+                0.16356658935546875,
+                0.16398643493652343,
+                0.16355020141601562,
+                0.1639024658203125,
+                0.1635952606201172,
+                0.1638707275390625,
+                0.16346112060546875,
+                0.1639086151123047,
+                0.16355328369140626,
+                0.16398541259765625,
+                0.16351231384277343,
+                0.16404173278808593,
+                0.16354713439941407,
+                0.16402330017089845,
+                0.16391168212890625,
+                0.16387890625,
+                0.16356761169433592,
+                0.16396815490722655,
+                0.16355226135253906,
+                0.16395468139648436,
+                0.16351849365234375,
+                0.16387992858886719,
+                0.16384512329101564,
+                0.16396902465820312,
+                0.16359219360351562,
+                0.16402249145507813,
+                0.1635379180908203,
+                0.16394546508789062,
+                0.16359834289550781,
+                0.16391783142089844,
+                0.16360243225097656,
+                0.1639078369140625,
+                0.16358195495605468,
+                0.16405821228027342,
+                0.16352767944335939,
+                0.1640069122314453,
+                0.16356454467773437,
+                0.16393215942382813,
+                0.16352359008789064,
+                0.1639772186279297,
+                0.1636444091796875,
+                0.16400588989257814
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 1561.5199495208317
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 4967.591936,
+            "max_vram": 4047.50336,
+            "max_reserved": 3563.061248,
+            "max_allocated": 3355.268608
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 8.078264411926268,
+            "stdev": 0.011069528579710664,
+            "values": [
+                8.089333940505979,
+                8.067194883346557
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 31.566186373341928
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.03167946828206381,
+            "stdev": 0.0001954214184208151,
+            "values": [
+                0.033990657806396485,
+                0.033116161346435545,
+                0.03187609672546387,
+                0.0317890567779541,
+                0.0318023681640625,
+                0.03180544090270996,
+                0.031734783172607424,
+                0.03168870353698731,
+                0.031591424942016604,
+                0.031647743225097655,
+                0.03169824028015137,
+                0.031758016586303714,
+                0.03138150405883789,
+                0.031307775497436525,
+                0.031237119674682616,
+                0.03167436790466309,
+                0.03165695953369141,
+                0.03172863960266113,
+                0.03165184020996094,
+                0.03167948722839355,
+                0.03171327972412109,
+                0.031704063415527346,
+                0.031698944091796875,
+                0.031720447540283206,
+                0.031667327880859374,
+                0.031913856506347656,
+                0.03159449577331543,
+                0.03170611190795898,
+                0.031547391891479495,
+                0.03163955116271973,
+                0.03168777656555176,
+                0.03173980712890625,
+                0.031733760833740236,
+                0.031748096466064454,
+                0.031698944091796875,
+                0.031716352462768556,
+                0.031734783172607424,
+                0.031953920364379884,
+                0.031665151596069335,
+                0.03152383995056152,
+                0.03167948722839355,
+                0.031579135894775394,
+                0.031612960815429685,
+                0.03166511917114258,
+                0.0318156795501709,
+                0.03169484710693359,
+                0.03179212760925293,
+                0.0316180477142334,
+                0.031681535720825195,
+                0.03167129516601563,
+                0.031710208892822264,
+                0.0317255687713623,
+                0.03158937644958496,
+                0.03156384086608887,
+                0.031449024200439456,
+                0.031710208892822264,
+                0.03169075202941894,
+                0.03155353546142578,
+                0.03160780715942383,
+                0.03175424003601074,
+                0.0316080322265625,
+                0.03177680015563965,
+                0.0317969913482666,
+                0.03156480026245117,
+                0.031647743225097655,
+                0.03166748809814453,
+                0.03172220802307129,
+                0.03177267265319824,
+                0.03183718490600586,
+                0.03178803253173828,
+                0.03162937545776367,
+                0.03172159957885742,
+                0.03175302314758301,
+                0.03165609550476074,
+                0.031666015625,
+                0.031638784408569334,
+                0.0317969913482666,
+                0.03178598403930664,
+                0.031753215789794925,
+                0.031693824768066405,
+                0.03170627212524414,
+                0.031756128311157227,
+                0.03176959991455078,
+                0.03151587104797363,
+                0.0315328311920166,
+                0.03238399887084961,
+                0.03271475219726563,
+                0.03202969741821289,
+                0.03168358421325684,
+                0.03169484710693359,
+                0.03173814392089844,
+                0.03160451126098633,
+                0.03159456062316895,
+                0.031695743560791016,
+                0.03163852882385254,
+                0.0317717113494873,
+                0.03174496078491211,
+                0.032074752807617186,
+                0.031718399047851564,
+                0.03176959991455078,
+                0.0317255687713623,
+                0.031704095840454104,
+                0.031724672317504886,
+                0.031532896041870116,
+                0.03170841598510742,
+                0.03165772819519043,
+                0.03177267265319824,
+                0.03171327972412109,
+                0.03266569519042969,
+                0.031852447509765625,
+                0.03165798377990723,
+                0.03160063934326172,
+                0.031747360229492184,
+                0.03164233589172363,
+                0.03167231941223145,
+                0.03175014305114746,
+                0.03182080078125,
+                0.031865856170654294,
+                0.031817728042602536,
+                0.031922176361083986,
+                0.03176755142211914,
+                0.03178188705444336,
+                0.03181977653503418,
+                0.03152822494506836,
+                0.031432735443115235,
+                0.03164640045166016,
+                0.03154022407531738,
+                0.03171356773376465,
+                0.032105342864990234,
+                0.032701278686523436,
+                0.032038047790527345,
+                0.03191689682006836,
+                0.031716352462768556,
+                0.0316866569519043,
+                0.031851520538330076,
+                0.03163750457763672,
+                0.03175014305114746,
+                0.03182796859741211,
+                0.03177574348449707,
+                0.03191193580627441,
+                0.03176038360595703,
+                0.03178291130065918,
+                0.03168870353698731,
+                0.031684736251831054,
+                0.031710079193115234,
+                0.03161926460266113,
+                0.03166496086120606,
+                0.03171225547790527,
+                0.03192831993103027,
+                0.03181670379638672,
+                0.03169177627563476,
+                0.031714303970336914,
+                0.031749120712280275,
+                0.031676416397094724,
+                0.03172659111022949,
+                0.03167129516601563,
+                0.03163033676147461,
+                0.03165388870239258,
+                0.03168889617919922,
+                0.031641408920288085,
+                0.0318023681640625,
+                0.03161087989807129,
+                0.031783935546875,
+                0.03176755142211914,
+                0.031665151596069335,
+                0.03159756851196289,
+                0.03146854400634766,
+                0.03162112045288086,
+                0.031604736328125,
+                0.03167129516601563,
+                0.0318474235534668,
+                0.03189555168151856,
+                0.03194572830200195,
+                0.03164675140380859,
+                0.031623136520385745,
+                0.03174604797363281,
+                0.03163750457763672,
+                0.031603872299194334,
+                0.03127180862426758,
+                0.03163337516784668,
+                0.031558656692504884,
+                0.031662080764770506,
+                0.03167334365844727,
+                0.03161702346801758,
+                0.0317063045501709,
+                0.03156153678894043,
+                0.03155251121520996,
+                0.03164672088623047,
+                0.03179315185546875,
+                0.031649791717529296,
+                0.03175628852844238,
+                0.03180646324157715,
+                0.03164284706115723,
+                0.03167440032958985,
+                0.031734527587890624,
+                0.031740095138549806,
+                0.03153900718688965,
+                0.03160883140563965,
+                0.031730688095092774,
+                0.03161523246765137,
+                0.031822591781616213,
+                0.03187814331054688,
+                0.03172249603271484,
+                0.03161702346801758,
+                0.031694944381713865,
+                0.031664031982421875,
+                0.031528959274291994,
+                0.03148185539245606,
+                0.03163545608520508,
+                0.03160883140563965,
+                0.031579135894775394,
+                0.031631359100341795,
+                0.031615999221801756,
+                0.0316753921508789,
+                0.03170508766174317,
+                0.031719423294067385,
+                0.031719423294067385,
+                0.03156480026245117,
+                0.031631359100341795,
+                0.031699968338012696,
+                0.03178803253173828,
+                0.03179315185546875,
+                0.03185971260070801,
+                0.03164672088623047,
+                0.03177267265319824,
+                0.03177471923828125,
+                0.03171327972412109,
+                0.031699968338012696,
+                0.031676416397094724,
+                0.031676416397094724,
+                0.031720447540283206,
+                0.031644672393798826,
+                0.0320552978515625,
+                0.03177574348449707,
+                0.031833087921142575,
+                0.03172761535644531,
+                0.03170611190795898,
+                0.03176140785217285,
+                0.031732736587524416,
+                0.03166332817077637,
+                0.03158505630493164,
+                0.03153817558288574,
+                0.031579135894775394,
+                0.03163347244262695,
+                0.03172447967529297,
+                0.03157811164855957,
+                0.031703039169311525,
+                0.0315545597076416,
+                0.03155558395385742,
+                0.03141244888305664,
+                0.031643423080444336,
+                0.03143075180053711,
+                0.03156572723388672,
+                0.03188531112670898,
+                0.03159161567687988,
+                0.03134464073181152,
+                0.03157516860961914,
+                0.03163347244262695,
+                0.031719423294067385,
+                0.03172640037536621,
+                0.031546367645263675,
+                0.03157606315612793,
+                0.03153408050537109,
+                0.03152179145812988,
+                0.031665151596069335,
+                0.03152614402770996,
+                0.03165670394897461,
+                0.03150540733337402,
+                0.031699968338012696,
+                0.03151155281066895,
+                0.03155148887634277,
+                0.031666175842285156,
+                0.03157625579833984,
+                0.03152979278564453,
+                0.03160576057434082,
+                0.03168358421325684,
+                0.031575103759765626,
+                0.031521728515625,
+                0.03156582450866699,
+                0.03156991958618164,
+                0.03169177627563476,
+                0.03155148887634277,
+                0.0316364803314209,
+                0.03155148887634277,
+                0.03165305519104004,
+                0.03155027198791504,
+                0.031521024703979494,
+                0.03155430412292481,
+                0.03159756851196289,
+                0.03153817558288574,
+                0.03165388870239258,
+                0.031531232833862305,
+                0.03146227264404297,
+                0.031614879608154296,
+                0.03174307250976562,
+                0.031479711532592776,
+                0.03165184020996094,
+                0.03158323287963867,
+                0.03148492813110351,
+                0.03156787109375,
+                0.03168563270568848,
+                0.03154022407531738,
+                0.03159654426574707,
+                0.03159449577331543,
+                0.03150643157958984,
+                0.031453439712524414,
+                0.03166713523864746,
+                0.03161580848693848,
+                0.031525888442993165,
+                0.03146342468261719,
+                0.0315883846282959,
+                0.03167024040222168,
+                0.03171327972412109,
+                0.03160371208190918,
+                0.03171737670898438,
+                0.03183539199829102,
+                0.031702783584594725,
+                0.03163033676147461,
+                0.03162931251525879,
+                0.03158537673950195,
+                0.031701183319091795,
+                0.031603488922119144,
+                0.031529920578002926,
+                0.03167948722839355,
+                0.031503360748291014,
+                0.03156480026245117,
+                0.03149516868591309,
+                0.031678464889526366,
+                0.031678464889526366,
+                0.031662080764770506,
+                0.031680511474609374,
+                0.03162009620666504,
+                0.03161408042907715,
+                0.03165068817138672,
+                0.03165097618103027,
+                0.03154518318176269,
+                0.03180963134765625,
+                0.03159542465209961,
+                0.031578399658203124,
+                0.03136079978942871,
+                0.03156681632995605,
+                0.031538143157958984,
+                0.03170099258422852,
+                0.032524288177490236,
+                0.031731712341308595,
+                0.03169587135314941,
+                0.031628288269042966,
+                0.03178803253173828,
+                0.031731807708740234,
+                0.031679391860961914,
+                0.03161497688293457,
+                0.03162009620666504,
+                0.03172761535644531,
+                0.031736831665039066,
+                0.031678464889526366,
+                0.03164060783386231,
+                0.03179209518432617,
+                0.03161292839050293,
+                0.03162931251525879,
+                0.03160166358947754,
+                0.03151974487304687,
+                0.03155046463012695,
+                0.03167334365844727,
+                0.031715328216552735,
+                0.03166838455200195,
+                0.0317684154510498,
+                0.03160166358947754,
+                0.03160268783569336,
+                0.03157606315612793,
+                0.031703039169311525,
+                0.03158323287963867,
+                0.031438848495483396,
+                0.03108358383178711,
+                0.031147968292236327,
+                0.031213567733764647,
+                0.031821823120117186,
+                0.03163955116271973,
+                0.03176345634460449,
+                0.03166924858093262,
+                0.03163545608520508,
+                0.03158016014099121,
+                0.03168563270568848,
+                0.031665151596069335,
+                0.031821823120117186,
+                0.031748096466064454,
+                0.03187331199645996,
+                0.03182665634155273,
+                0.03188659286499024,
+                0.03189017677307129,
+                0.03169177627563476,
+                0.03172672080993652,
+                0.03175820732116699,
+                0.03169484710693359,
+                0.03165705680847168,
+                0.03159440040588379,
+                0.03154841613769531,
+                0.03150540733337402,
+                0.031502496719360354,
+                0.03166703987121582,
+                0.03160371208190918,
+                0.03162419128417969,
+                0.031628288269042966,
+                0.03148185539245606,
+                0.03181158447265625,
+                0.031749120712280275,
+                0.03172761535644531,
+                0.03175014305114746,
+                0.03168460845947266,
+                0.03156172752380371,
+                0.031507455825805664,
+                0.031647008895874025,
+                0.03155324745178223,
+                0.031469791412353516,
+                0.03161372756958008,
+                0.03168870353698731,
+                0.03159859275817871,
+                0.03161500740051269,
+                0.031594463348388675,
+                0.031704095840454104,
+                0.03163132858276367,
+                0.031648767471313476,
+                0.03169484710693359,
+                0.031838207244873046,
+                0.03154841613769531,
+                0.03174710464477539,
+                0.0316978874206543,
+                0.03174211120605469,
+                0.03177456092834473,
+                0.03170816040039062,
+                0.03172966384887695,
+                0.031635488510131836,
+                0.03160367965698242,
+                0.031719423294067385,
+                0.03172659111022949,
+                0.031692800521850584,
+                0.03168870353698731,
+                0.03176470375061035,
+                0.03183798408508301,
+                0.03166924858093262,
+                0.031720447540283206,
+                0.0316549129486084,
+                0.03170611190795898,
+                0.031848703384399414,
+                0.03169055938720703,
+                0.031723455429077146,
+                0.03176345634460449,
+                0.03169484710693359,
+                0.031942655563354495,
+                0.03190169525146484,
+                0.031753215789794925,
+                0.03173990440368652,
+                0.031720447540283206,
+                0.03171737670898438,
+                0.031753215789794925,
+                0.0317696647644043,
+                0.03197542381286621,
+                0.03164255905151367,
+                0.03175833511352539,
+                0.031714303970336914,
+                0.031663103103637694,
+                0.031615999221801756,
+                0.03160268783569336,
+                0.03169484710693359,
+                0.03162112045288086,
+                0.03160585594177246,
+                0.031648672103881836,
+                0.03165184020996094,
+                0.03171443176269531,
+                0.03161075210571289,
+                0.03158118438720703,
+                0.03169804763793945,
+                0.03151142311096192,
+                0.03150035285949707,
+                0.03169401550292969,
+                0.031634464263916015,
+                0.031678176879882815,
+                0.031646816253662106,
+                0.03153091239929199,
+                0.03139686393737793,
+                0.031541248321533204,
+                0.03165286445617676,
+                0.03168767929077149,
+                0.03150131225585937,
+                0.03157196807861328,
+                0.03156070327758789,
+                0.031473663330078124,
+                0.03150643157958984,
+                0.03186892890930176,
+                0.0315545597076416,
+                0.03176959991455078,
+                0.03160883140563965,
+                0.031648767471313476,
+                0.03167948722839355,
+                0.03178803253173828,
+                0.031703039169311525,
+                0.03166924858093262,
+                0.03157606315612793,
+                0.03163868713378906,
+                0.03160560035705567,
+                0.03168767929077149,
+                0.03161907196044922,
+                0.0316180477142334,
+                0.03165081596374512,
+                0.03150131225585937,
+                0.0314113597869873,
+                0.03164348793029785,
+                0.03141529655456543,
+                0.03152179145812988,
+                0.03133443260192871,
+                0.03130364799499512
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 31.56618637334191
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}