Upload /Llama-2-7b-hf/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

Llama-2-7b-hf/fp4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,811 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 982.437888,
+            "max_vram": 4905.238528,
+            "max_reserved": 4420.796416,
+            "max_allocated": 4252.667392
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.042258727737619926,
+            "stdev": 0.000582065656107356,
+            "values": [
+                0.047258270263671874,
+                0.04455014419555664,
+                0.04322304153442383,
+                0.043133953094482425,
+                0.04293443298339844,
+                0.04335615921020508,
+                0.04328345489501953,
+                0.04333465576171875,
+                0.04698521423339844,
+                0.04396953582763672,
+                0.04190105438232422,
+                0.04237209701538086,
+                0.0419502067565918,
+                0.042387454986572266,
+                0.041899009704589846,
+                0.042355712890625,
+                0.041852928161621096,
+                0.042379264831542966,
+                0.04192870330810547,
+                0.04236492919921875,
+                0.04185190582275391,
+                0.04237209701538086,
+                0.04184371185302734,
+                0.04237823867797851,
+                0.04193689727783203,
+                0.04254924774169922,
+                0.04188979339599609,
+                0.04238643264770508,
+                0.04189491271972656,
+                0.042365951538085936,
+                0.04185513687133789,
+                0.04237516784667969,
+                0.04209664154052734,
+                0.04235161590576172,
+                0.04186111831665039,
+                0.04240486526489258,
+                0.04228607940673828,
+                0.04238336181640625,
+                0.0418785285949707,
+                0.0424007682800293,
+                0.04184678268432617,
+                0.042349567413330076,
+                0.041859073638916014,
+                0.04237020874023437,
+                0.04184473419189453,
+                0.042382335662841795,
+                0.04188159942626953,
+                0.04237120056152344,
+                0.041855998992919925,
+                0.04238336181640625,
+                0.041867263793945314,
+                0.042355712890625,
+                0.04187648010253906,
+                0.042349567413330076,
+                0.041847808837890625,
+                0.042398719787597655,
+                0.04206796646118164,
+                0.04261478424072265,
+                0.04187955093383789,
+                0.04237721633911133,
+                0.041864192962646485,
+                0.042363903045654294,
+                0.04187750244140625,
+                0.042398719787597655,
+                0.04185804748535156,
+                0.04237635040283203,
+                0.04187238311767578,
+                0.04236207962036133,
+                0.041864192962646485,
+                0.042355712890625,
+                0.04189184188842773,
+                0.042387454986572266,
+                0.04232294464111328,
+                0.04237311935424805,
+                0.041866241455078126,
+                0.042369022369384765,
+                0.04188671875,
+                0.042343425750732425,
+                0.04193689727783203,
+                0.0423454704284668,
+                0.04187955093383789,
+                0.042349567413330076,
+                0.04190003204345703,
+                0.042355712890625,
+                0.04231270217895508,
+                0.04235673522949219,
+                0.04196681594848633,
+                0.04238438415527344,
+                0.041866241455078126,
+                0.042369022369384765,
+                0.04189388656616211,
+                0.0423785285949707,
+                0.04193689727783203,
+                0.042359809875488284,
+                0.0419317741394043,
+                0.042379264831542966,
+                0.04190719985961914,
+                0.04236185455322266,
+                0.04187340927124023,
+                0.04235161590576172,
+                0.04186111831665039,
+                0.04235366439819336,
+                0.04187443161010742,
+                0.042347518920898435,
+                0.04185190582275391,
+                0.04238556671142578,
+                0.04185702514648437,
+                0.04236492919921875,
+                0.04186316680908203,
+                0.042363903045654294,
+                0.041856224060058594,
+                0.04234649658203125,
+                0.0418950080871582,
+                0.04234239959716797,
+                0.041918464660644535,
+                0.042369022369384765,
+                0.04185804748535156,
+                0.04236083221435547,
+                0.04193280029296875,
+                0.04235776138305664,
+                0.04231475067138672,
+                0.0423741455078125,
+                0.0419420166015625,
+                0.04237619018554688,
+                0.04199321746826172,
+                0.04238131332397461,
+                0.04203212738037109,
+                0.042393600463867184,
+                0.04202598571777344,
+                0.04235776138305664,
+                0.04187750244140625,
+                0.04236083221435547,
+                0.04218265533447266,
+                0.042385406494140625,
+                0.041952255249023435,
+                0.04235776138305664,
+                0.041902080535888675,
+                0.04237516784667969,
+                0.0418600959777832,
+                0.04236697769165039,
+                0.042014720916748044,
+                0.04235161590576172,
+                0.04187033462524414,
+                0.04235673522949219,
+                0.04192256164550781,
+                0.04237721633911133,
+                0.04187648010253906,
+                0.042343425750732425,
+                0.0419051513671875,
+                0.0423741455078125,
+                0.04198604965209961,
+                0.04236288070678711,
+                0.04187648010253906,
+                0.04237516784667969,
+                0.04206694412231445,
+                0.04235161590576172,
+                0.04191436767578125,
+                0.044368896484375,
+                0.04288819122314453,
+                0.0423741455078125,
+                0.041855998992919925,
+                0.04235468673706055,
+                0.041896961212158204,
+                0.04238336181640625,
+                0.04206284713745117,
+                0.042355712890625,
+                0.04192051315307617,
+                0.04234035110473633,
+                0.04248473739624024,
+                0.04237619018554688,
+                0.04198809432983398,
+                0.042365951538085936,
+                0.042014942169189454,
+                0.04236083221435547,
+                0.04184473419189453,
+                0.04235366439819336,
+                0.04186316680908203,
+                0.042401790618896484,
+                0.042354782104492186,
+                0.04240691375732422,
+                0.04187136077880859,
+                0.042374305725097657,
+                0.04206911849975586,
+                0.042365119934082034,
+                0.04197580718994141,
+                0.042363903045654294,
+                0.04193382263183594,
+                0.042387454986572266,
+                0.04192563247680664,
+                0.04235878372192383,
+                0.04187750244140625,
+                0.04238336181640625,
+                0.04209356689453125,
+                0.04237619018554688,
+                0.041957374572753905,
+                0.04233932876586914,
+                0.04196454238891602,
+                0.04238848114013672,
+                0.04195635223388672,
+                0.04237311935424805,
+                0.04188979339599609,
+                0.04236185455322266,
+                0.041968639373779294,
+                0.04237619018554688,
+                0.041902080535888675,
+                0.04238051223754883,
+                0.042036479949951175,
+                0.042349567413330076,
+                0.041918464660644535,
+                0.04236185455322266,
+                0.041968639373779294,
+                0.04236083221435547,
+                0.041908222198486327,
+                0.04236185455322266,
+                0.041855998992919925,
+                0.042379264831542966,
+                0.042425342559814457,
+                0.04237516784667969,
+                0.04187750244140625,
+                0.042369281768798825,
+                0.04196044921875,
+                0.04237619018554688,
+                0.04188703918457031,
+                0.04237619018554688,
+                0.04194105529785156,
+                0.042382335662841795,
+                0.042172416687011716,
+                0.04245196914672852,
+                0.04191027069091797,
+                0.0423454704284668,
+                0.042281982421875,
+                0.04236697769165039,
+                0.04186316680908203,
+                0.042355712890625,
+                0.04190310287475586,
+                0.04237619018554688,
+                0.04189593505859375
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 6057.920190817801
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 982.437888,
+            "max_vram": 5255.462912,
+            "max_reserved": 4771.0208,
+            "max_allocated": 4715.611648
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 8.489722272872921,
+            "stdev": 0.012863920211795055,
+            "values": [
+                8.476858352661127,
+                8.502586193084717
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 30.03631824503819
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.03329302852107029,
+            "stdev": 0.0001376279300769796,
+            "values": [
+                0.03315507125854492,
+                0.03353497695922852,
+                0.03332505416870117,
+                0.03326591873168945,
+                0.033143550872802734,
+                0.033260639190673826,
+                0.03330140686035156,
+                0.03323904037475586,
+                0.03323494338989258,
+                0.033186817169189455,
+                0.033143806457519534,
+                0.03326668930053711,
+                0.03321139144897461,
+                0.03326976013183594,
+                0.03401318359375,
+                0.0333007698059082,
+                0.03305648040771485,
+                0.033040382385253905,
+                0.033113086700439456,
+                0.03319091033935547,
+                0.03304243087768555,
+                0.03308236694335937,
+                0.03324518585205078,
+                0.03311820983886719,
+                0.033124351501464845,
+                0.03313868713378906,
+                0.03303833770751953,
+                0.03317452621459961,
+                0.03328921508789062,
+                0.03322265625,
+                0.033165313720703124,
+                0.03322367858886719,
+                0.03372851181030274,
+                0.033185791015625,
+                0.03314995193481445,
+                0.03333529663085937,
+                0.033186817169189455,
+                0.03325030517578125,
+                0.03328921508789062,
+                0.033260543823242186,
+                0.0332410888671875,
+                0.03316534423828125,
+                0.03322364807128906,
+                0.03316633605957031,
+                0.033172481536865236,
+                0.03330998229980469,
+                0.03313123321533203,
+                0.03381862258911133,
+                0.03307110214233398,
+                0.0331960334777832,
+                0.03316659164428711,
+                0.033193824768066406,
+                0.03315292739868164,
+                0.03325747299194336,
+                0.03307622528076172,
+                0.033031169891357424,
+                0.03310899353027344,
+                0.03341843032836914,
+                0.03297977447509766,
+                0.03315302276611328,
+                0.033127422332763674,
+                0.03314483261108398,
+                0.03308342361450195,
+                0.033077217102050784,
+                0.033291263580322264,
+                0.03320115280151367,
+                0.033083393096923826,
+                0.033178752899169925,
+                0.03321433639526367,
+                0.033209342956542966,
+                0.03310182571411133,
+                0.03337625503540039,
+                0.03318476867675781,
+                0.033438751220703125,
+                0.03312432098388672,
+                0.03304243087768555,
+                0.03305267333984375,
+                0.033143806457519534,
+                0.033058815002441407,
+                0.03324825668334961,
+                0.03322777557373047,
+                0.033233951568603516,
+                0.03321136093139648,
+                0.03331071853637695,
+                0.033350654602050785,
+                0.033279998779296875,
+                0.03313151931762695,
+                0.03329740905761719,
+                0.03326464080810547,
+                0.03327385711669922,
+                0.03329536056518555,
+                0.03327391815185547,
+                0.03313555145263672,
+                0.03367424011230469,
+                0.0333199348449707,
+                0.03324518585205078,
+                0.033175552368164066,
+                0.03320124816894531,
+                0.03319696044921875,
+                0.03317657470703125,
+                0.03317145538330078,
+                0.03331071853637695,
+                0.033339393615722655,
+                0.03322060775756836,
+                0.03315507125854492,
+                0.03337539291381836,
+                0.0333812141418457,
+                0.03341209411621094,
+                0.033486846923828126,
+                0.03326464080810547,
+                0.03314995193481445,
+                0.03321343994140625,
+                0.03326668930053711,
+                0.033337345123291014,
+                0.0335093765258789,
+                0.03333657455444336,
+                0.03333504104614258,
+                0.03328409576416016,
+                0.03325132751464844,
+                0.0332861442565918,
+                0.03321446228027344,
+                0.033334270477294925,
+                0.03328409576416016,
+                0.033364158630371094,
+                0.03328799819946289,
+                0.033396736145019534,
+                0.03326566314697266,
+                0.03334041595458984,
+                0.033331199645996096,
+                0.03330355072021484,
+                0.033160190582275394,
+                0.03320537567138672,
+                0.03315596771240235,
+                0.03327283096313476,
+                0.03318067169189453,
+                0.033323009490966796,
+                0.03323187255859375,
+                0.03327385711669922,
+                0.03320729446411133,
+                0.03323289489746094,
+                0.033293312072753906,
+                0.03315302276611328,
+                0.03324620819091797,
+                0.033219585418701174,
+                0.033192958831787106,
+                0.03319091033935547,
+                0.0331776008605957,
+                0.033271808624267575,
+                0.033135616302490234,
+                0.033175552368164066,
+                0.033175552368164066,
+                0.033323009490966796,
+                0.033119232177734374,
+                0.03321753692626953,
+                0.03366400146484375,
+                0.033230846405029296,
+                0.03325132751464844,
+                0.033462272644042966,
+                0.033230846405029296,
+                0.03319091033935547,
+                0.03315609741210938,
+                0.03331379318237305,
+                0.0332042236328125,
+                0.033328128814697267,
+                0.033301502227783206,
+                0.03336908721923828,
+                0.03324313735961914,
+                0.03333529663085937,
+                0.033667072296142575,
+                0.03329740905761719,
+                0.033463294982910154,
+                0.033235233306884764,
+                0.033362655639648436,
+                0.03308544158935547,
+                0.03311718368530273,
+                0.03316428756713867,
+                0.03306496047973633,
+                0.03307827377319336,
+                0.0331141128540039,
+                0.03307417678833008,
+                0.03305984115600586,
+                0.03318067169189453,
+                0.03315507125854492,
+                0.032982017517089846,
+                0.03300864028930664,
+                0.03314790344238281,
+                0.03309875106811523,
+                0.033007614135742186,
+                0.03304857635498047,
+                0.03325747299194336,
+                0.03307417678833008,
+                0.033170433044433595,
+                0.03322675323486328,
+                0.03318476867675781,
+                0.03311206436157227,
+                0.033188865661621096,
+                0.03305984115600586,
+                0.03315609741210938,
+                0.033132545471191405,
+                0.0331776008605957,
+                0.033438720703125,
+                0.03308441543579101,
+                0.033091583251953126,
+                0.033107967376708985,
+                0.03325439834594727,
+                0.033209342956542966,
+                0.0331253776550293,
+                0.03307417678833008,
+                0.033068031311035154,
+                0.0332564468383789,
+                0.03313868713378906,
+                0.033142784118652346,
+                0.03304652786254883,
+                0.033113086700439456,
+                0.03360665512084961,
+                0.033410049438476565,
+                0.033119232177734374,
+                0.03349200057983399,
+                0.033191902160644535,
+                0.03338979339599609,
+                0.033137439727783206,
+                0.033320960998535154,
+                0.03320627212524414,
+                0.03322367858886719,
+                0.033279998779296875,
+                0.03355347061157227,
+                0.03323897552490234,
+                0.03323801422119141,
+                0.033235969543457033,
+                0.03385036849975586,
+                0.033307647705078124,
+                0.03336601638793945,
+                0.03354521560668945,
+                0.033358848571777344,
+                0.03334348678588867,
+                0.03326985549926758,
+                0.03338435363769531,
+                0.0332410888671875,
+                0.03324313735961914,
+                0.0332564468383789,
+                0.03326976013183594,
+                0.03331398391723633,
+                0.03333407974243164,
+                0.033331199645996096,
+                0.033379520416259766,
+                0.033417022705078125,
+                0.0332492790222168,
+                0.033432575225830076,
+                0.03322470474243164,
+                0.03325337600708008,
+                0.03328409576416016,
+                0.03337955093383789,
+                0.03337705612182617,
+                0.03328409576416016,
+                0.03338547134399414,
+                0.03331875228881836,
+                0.033247230529785156,
+                0.03322060775756836,
+                0.03324739074707031,
+                0.0332655029296875,
+                0.03320729446411133,
+                0.033255424499511715,
+                0.03323113632202149,
+                0.03315683364868164,
+                0.03327385711669922,
+                0.033242111206054685,
+                0.03320012664794922,
+                0.03324620819091797,
+                0.033268798828125,
+                0.03313657760620117,
+                0.0332760009765625,
+                0.03317238235473633,
+                0.03328118515014648,
+                0.033259361267089844,
+                0.03370195388793945,
+                0.033266624450683596,
+                0.03314176177978516,
+                0.03318374252319336,
+                0.033308830261230465,
+                0.03329315185546875,
+                0.03330047988891602,
+                0.0333199348449707,
+                0.03330876922607422,
+                0.033288097381591795,
+                0.03336294555664063,
+                0.03331174468994141,
+                0.03337830352783203,
+                0.03330252838134766,
+                0.033643585205078125,
+                0.033210304260253905,
+                0.03321343994140625,
+                0.03325439834594727,
+                0.03336908721923828,
+                0.033261566162109374,
+                0.033315135955810544,
+                0.03331961441040039,
+                0.03327283096313476,
+                0.03336191940307617,
+                0.03353497695922852,
+                0.03325132751464844,
+                0.033312767028808594,
+                0.033263614654541016,
+                0.03340800094604492,
+                0.03316428756713867,
+                0.03330047988891602,
+                0.03322982406616211,
+                0.03336115264892578,
+                0.033366783142089844,
+                0.03330047988891602,
+                0.03331686401367188,
+                0.033428577423095705,
+                0.03340995025634766,
+                0.033363967895507815,
+                0.0332042236328125,
+                0.03325337600708008,
+                0.03333017730712891,
+                0.033261566162109374,
+                0.03321036911010742,
+                0.03328102493286133,
+                0.03342950439453125,
+                0.033328128814697267,
+                0.033296382904052735,
+                0.03334873580932617,
+                0.033309566497802735,
+                0.033329151153564454,
+                0.033331199645996096,
+                0.03341209411621094,
+                0.03345510482788086,
+                0.033347583770751955,
+                0.03336703872680664,
+                0.03339571380615235,
+                0.03342335891723633,
+                0.0333199348449707,
+                0.0333199348449707,
+                0.03384012985229492,
+                0.03338444900512695,
+                0.03344179153442383,
+                0.03342233657836914,
+                0.03335372924804687,
+                0.03339980697631836,
+                0.03345510482788086,
+                0.03331379318237305,
+                0.03341721725463867,
+                0.03334860610961914,
+                0.03377260971069336,
+                0.033310657501220704,
+                0.03338751983642578,
+                0.033342464447021485,
+                0.03351551818847656,
+                0.033529857635498046,
+                0.03349708938598633,
+                0.033317886352539065,
+                0.03347260665893555,
+                0.033340320587158204,
+                0.03333529663085937,
+                0.03331379318237305,
+                0.03337420654296875,
+                0.03335782241821289,
+                0.033430782318115235,
+                0.03330227279663086,
+                0.03347251129150391,
+                0.033344512939453126,
+                0.03339571380615235,
+                0.033277950286865234,
+                0.033274879455566404,
+                0.033329151153564454,
+                0.03336191940307617,
+                0.03326259231567383,
+                0.03332710266113281,
+                0.03323904037475586,
+                0.03338854217529297,
+                0.033285118103027346,
+                0.03341721725463867,
+                0.03327078247070313,
+                0.03327078247070313,
+                0.033312767028808594,
+                0.033372161865234375,
+                0.0333834228515625,
+                0.0334284782409668,
+                0.033339393615722655,
+                0.03341107177734375,
+                0.03328841781616211,
+                0.03335248184204102,
+                0.033309696197509765,
+                0.033230846405029296,
+                0.033344512939453126,
+                0.03335372924804687,
+                0.03332505416870117,
+                0.03328204727172852,
+                0.03323699188232422,
+                0.033533119201660154,
+                0.03339961624145508,
+                0.03322982406616211,
+                0.03334860610961914,
+                0.03352371215820313,
+                0.033620990753173825,
+                0.03331071853637695,
+                0.033285118103027346,
+                0.03344076919555664,
+                0.033430526733398434,
+                0.03321651077270508,
+                0.03342335891723633,
+                0.03354521560668945,
+                0.03337932968139649,
+                0.03327590560913086,
+                0.033304576873779294,
+                0.03339980697631836,
+                0.033258495330810545,
+                0.0334919662475586,
+                0.03374387359619141,
+                0.033426433563232424,
+                0.03347148895263672,
+                0.03333126449584961,
+                0.03327686309814453,
+                0.033350654602050785,
+                0.033342464447021485,
+                0.03321241760253906,
+                0.033290241241455076,
+                0.03330572891235352,
+                0.033309566497802735,
+                0.03338956832885742,
+                0.03322777557373047,
+                0.033486846923828126,
+                0.03322777557373047,
+                0.03335391998291016,
+                0.033289024353027344,
+                0.033495166778564456,
+                0.033320831298828124,
+                0.0332861442565918,
+                0.033309696197509765,
+                0.03337625503540039,
+                0.03327385711669922,
+                0.0333199348449707,
+                0.033288192749023435,
+                0.03337113571166992,
+                0.033271808624267575,
+                0.03324620819091797,
+                0.03320012664794922,
+                0.033274879455566404,
+                0.03336806488037109,
+                0.03329228973388672,
+                0.03322470474243164,
+                0.03323904037475586,
+                0.03333529663085937,
+                0.03325747299194336,
+                0.033432575225830076,
+                0.033317886352539065,
+                0.03331379318237305,
+                0.03335168075561523,
+                0.033375232696533204,
+                0.033345535278320314,
+                0.03330559921264648,
+                0.03334656143188477,
+                0.03329536056518555,
+                0.033301502227783206,
+                0.03385036849975586,
+                0.03324415969848633,
+                0.03343462371826172,
+                0.03332710266113281,
+                0.03319091033935547,
+                0.03343360137939453,
+                0.03332403182983398,
+                0.03321036911010742,
+                0.03340185546875,
+                0.033320960998535154,
+                0.03324518585205078,
+                0.033432575225830076,
+                0.033326080322265625,
+                0.03335168075561523,
+                0.03380223846435547,
+                0.03327590560913086,
+                0.033306625366210936,
+                0.03338137435913086,
+                0.033416191101074216,
+                0.033442047119140624,
+                0.03320191955566406,
+                0.03322265625,
+                0.03342131042480469,
+                0.03347558212280274,
+                0.03333631896972656,
+                0.03323801422119141,
+                0.033255424499511715,
+                0.033296382904052735,
+                0.03349708938598633,
+                0.03344179153442383,
+                0.03343360137939453,
+                0.033358848571777344,
+                0.03338444900512695,
+                0.03333324813842774,
+                0.033342464447021485,
+                0.03333017730712891,
+                0.0334284782409668,
+                0.0334284782409668,
+                0.03329945755004883,
+                0.033271808624267575,
+                0.033173503875732424,
+                0.03332505416870117,
+                0.03348992156982422,
+                0.03328716659545899,
+                0.03332403182983398,
+                0.033219585418701174,
+                0.03335782241821289,
+                0.033339393615722655,
+                0.03322470474243164,
+                0.033423553466796874,
+                0.03352659225463867,
+                0.033291263580322264,
+                0.033478656768798826,
+                0.03340083312988281,
+                0.03338649749755859
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 30.036318245038174
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}