Upload /phi-2/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

phi-2/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,845 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 964.23936,
+            "max_vram": 2585.788416,
+            "max_reserved": 2101.346304,
+            "max_allocated": 2014.912
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.03698565444383237,
+            "stdev": 0.0004510177520171469,
+            "values": [
+                0.040046463012695314,
+                0.03841535949707031,
+                0.03833446502685547,
+                0.037953536987304685,
+                0.03789823913574219,
+                0.03812454223632813,
+                0.03973961639404297,
+                0.041866241455078126,
+                0.03862220764160156,
+                0.03701657485961914,
+                0.03690188980102539,
+                0.036831230163574216,
+                0.03707699203491211,
+                0.03682304000854492,
+                0.03681280136108398,
+                0.03687321472167969,
+                0.03697868728637695,
+                0.036898815155029296,
+                0.03687526321411133,
+                0.03678617477416992,
+                0.03687526321411133,
+                0.03685990524291992,
+                0.036908031463623044,
+                0.036738048553466796,
+                0.03678515243530273,
+                0.036841472625732424,
+                0.03693875122070313,
+                0.036853759765625,
+                0.03682611083984375,
+                0.03723468780517578,
+                0.036794654846191405,
+                0.0368721923828125,
+                0.03683327865600586,
+                0.03692748641967773,
+                0.03684044647216797,
+                0.03696640014648438,
+                0.036931583404541016,
+                0.03688959884643555,
+                0.03674521636962891,
+                0.03685068893432617,
+                0.03673212814331055,
+                0.036790271759033204,
+                0.03718860626220703,
+                0.03680972671508789,
+                0.03685068893432617,
+                0.03680255889892578,
+                0.03683647918701172,
+                0.03683225631713867,
+                0.03686502456665039,
+                0.03682099151611328,
+                0.03683430480957031,
+                0.036918270111083985,
+                0.036953086853027346,
+                0.036837406158447265,
+                0.03681792068481445,
+                0.03687936019897461,
+                0.03683327865600586,
+                0.03678003311157227,
+                0.036736000061035154,
+                0.03679641723632812,
+                0.03672780990600586,
+                0.03685683059692383,
+                0.036743167877197266,
+                0.03676979064941406,
+                0.036760574340820314,
+                0.036775070190429686,
+                0.03673088073730469,
+                0.03686195373535156,
+                0.03670540618896485,
+                0.0368353271484375,
+                0.03673395156860351,
+                0.036822017669677735,
+                0.03672694396972656,
+                0.03673395156860351,
+                0.036770816802978515,
+                0.03678003311157227,
+                0.036746559143066404,
+                0.036915199279785156,
+                0.03684966278076172,
+                0.03700326538085937,
+                0.03687334442138672,
+                0.036746463775634765,
+                0.03687321472167969,
+                0.036926464080810545,
+                0.03707392120361328,
+                0.03678822326660156,
+                0.03692748641967773,
+                0.03681792068481445,
+                0.03684249496459961,
+                0.03679743957519531,
+                0.03690598297119141,
+                0.03681894302368164,
+                0.03674828720092774,
+                0.03670016098022461,
+                0.03678335952758789,
+                0.036751361846923826,
+                0.03678515243530273,
+                0.037233665466308595,
+                0.03681792068481445,
+                0.03680665588378906,
+                0.03683942413330078,
+                0.03688959884643555,
+                0.0369758415222168,
+                0.03681587219238281,
+                0.03684659194946289,
+                0.03684044647216797,
+                0.03679743957519531,
+                0.03693772888183594,
+                0.03684249496459961,
+                0.03684044647216797,
+                0.03676691055297852,
+                0.036896766662597655,
+                0.03690291213989258,
+                0.036907009124755856,
+                0.03685068893432617,
+                0.036928512573242187,
+                0.03695513534545898,
+                0.03693056106567383,
+                0.036841472625732424,
+                0.037010433197021485,
+                0.037043201446533204,
+                0.036972545623779295,
+                0.03690086364746094,
+                0.03720294570922852,
+                0.036942848205566405,
+                0.036877311706542966,
+                0.03702783966064453,
+                0.03688044738769531,
+                0.03695206451416016,
+                0.03707187271118164,
+                0.03698175811767578,
+                0.036939777374267575,
+                0.036955455780029296,
+                0.03700121688842774,
+                0.03704729461669922,
+                0.036913150787353514,
+                0.03693363189697266,
+                0.03696230316162109,
+                0.03743743896484375,
+                0.0369879035949707,
+                0.03694387054443359,
+                0.03690496063232422,
+                0.03689471817016601,
+                0.03695206451416016,
+                0.03691417694091797,
+                0.03701760101318359,
+                0.03696537780761719,
+                0.03693260955810547,
+                0.036929534912109374,
+                0.03696230316162109,
+                0.03689779281616211,
+                0.03729126358032227,
+                0.03690496063232422,
+                0.03697049713134765,
+                0.03683737564086914,
+                0.03694079971313476,
+                0.03696246337890625,
+                0.03694899368286133,
+                0.03688140869140625,
+                0.03694287872314453,
+                0.036890625,
+                0.03692134475708008,
+                0.0368721923828125,
+                0.03693875122070313,
+                0.03711795043945312,
+                0.036944896697998046,
+                0.03687321472167969,
+                0.03695513534545898,
+                0.03684966278076172,
+                0.03696640014648438,
+                0.03687936019897461,
+                0.037009471893310546,
+                0.0369541130065918,
+                0.03693670272827149,
+                0.03695206451416016,
+                0.03685171127319336,
+                0.03684249496459961,
+                0.03687628936767578,
+                0.03695743942260742,
+                0.036877311706542966,
+                0.036888671875,
+                0.03690496063232422,
+                0.03697356796264648,
+                0.036959232330322264,
+                0.036880382537841795,
+                0.036951038360595705,
+                0.03682611083984375,
+                0.03709235382080078,
+                0.03683635330200195,
+                0.036805633544921876,
+                0.03687936019897461,
+                0.03708313751220703,
+                0.036980735778808595,
+                0.03728998565673828,
+                0.03689984130859375,
+                0.03685068893432617,
+                0.036961280822753906,
+                0.03691417694091797,
+                0.036923393249511716,
+                0.036951038360595705,
+                0.03691622543334961,
+                0.036893695831298826,
+                0.036907009124755856,
+                0.03694825744628906,
+                0.036873279571533205,
+                0.03694182586669922,
+                0.037400577545166014,
+                0.036947967529296875,
+                0.036910079956054685,
+                0.03704627227783203,
+                0.03706675338745117,
+                0.03695820617675781,
+                0.036944896697998046,
+                0.0369541130065918,
+                0.03698387145996094,
+                0.03704422378540039,
+                0.03685068893432617,
+                0.03691017532348633,
+                0.036822017669677735,
+                0.0369879035949707,
+                0.03681792068481445,
+                0.036790271759033204,
+                0.03691417694091797,
+                0.037116928100585936,
+                0.03694387054443359,
+                0.03694694519042969,
+                0.036945919036865234,
+                0.03712847900390625,
+                0.036994049072265625,
+                0.0370882568359375,
+                0.03686406326293945,
+                0.037015552520751956,
+                0.03693056106567383,
+                0.03708848190307617,
+                0.036890625,
+                0.03697987365722656,
+                0.036896766662597655,
+                0.036969470977783206,
+                0.03686940765380859,
+                0.03697151947021484,
+                0.03685990524291992,
+                0.03694204711914063,
+                0.03697049713134765,
+                0.036985855102539066,
+                0.03686195373535156,
+                0.0369541130065918,
+                0.036898815155029296,
+                0.03737702560424805,
+                0.03688243103027344,
+                0.037043201446533204,
+                0.037043201446533204,
+                0.03700838470458984,
+                0.03710275268554687,
+                0.036951038360595705,
+                0.03696537780761719,
+                0.036980735778808595,
+                0.03701862335205078,
+                0.036994049072265625,
+                0.03691622543334961,
+                0.03693670272827149,
+                0.03731455993652344,
+                0.036983806610107424,
+                0.03691219329833984,
+                0.036915199279785156,
+                0.03697663879394531,
+                0.03694387054443359,
+                0.03688243103027344,
+                0.037026817321777344,
+                0.036947967529296875,
+                0.036928512573242187,
+                0.036982784271240236
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 6921.6025469758815
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 964.23936,
+            "max_vram": 2858.418176,
+            "max_reserved": 2373.976064,
+            "max_allocated": 2326.085632
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 7.248771577835086,
+            "stdev": 0.04500223922729729,
+            "values": [
+                7.203769338607789,
+                7.293773817062384
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 35.17837433031076
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.028426555207196384,
+            "stdev": 0.0007068028114393512,
+            "values": [
+                0.028911615371704103,
+                0.028435455322265626,
+                0.028400640487670898,
+                0.028308479309082032,
+                0.02835763168334961,
+                0.0283504638671875,
+                0.028325887680053712,
+                0.02835353660583496,
+                0.028296192169189452,
+                0.02830438423156738,
+                0.028318719863891603,
+                0.028263423919677736,
+                0.02831667137145996,
+                0.02830745506286621,
+                0.028244991302490235,
+                0.028421119689941408,
+                0.02831564712524414,
+                0.028310527801513673,
+                0.028403711318969727,
+                0.02833510398864746,
+                0.028229631423950196,
+                0.028314624786376953,
+                0.028265632629394532,
+                0.028312416076660157,
+                0.028291072845458985,
+                0.0283371524810791,
+                0.028277759552001954,
+                0.02832793617248535,
+                0.02834124755859375,
+                0.028399616241455077,
+                0.028309503555297853,
+                0.02833510398864746,
+                0.028483583450317384,
+                0.028447744369506835,
+                0.028280832290649413,
+                0.028313600540161132,
+                0.02834841537475586,
+                0.02833919906616211,
+                0.02833510398864746,
+                0.028647424697875977,
+                0.028290111541748045,
+                0.02844051170349121,
+                0.028253183364868165,
+                0.02838425636291504,
+                0.028416000366210937,
+                0.028358655929565428,
+                0.02830335998535156,
+                0.028279808044433592,
+                0.02836582374572754,
+                0.028308479309082032,
+                0.028267520904541016,
+                0.028564479827880858,
+                0.028278783798217775,
+                0.028392448425292968,
+                0.028222463607788087,
+                0.02819584083557129,
+                0.02854092788696289,
+                0.02813132858276367,
+                0.028096511840820314,
+                0.0281395206451416,
+                0.028073984146118162,
+                0.028189695358276368,
+                0.028215295791625978,
+                0.028226560592651367,
+                0.028237823486328126,
+                0.02812620735168457,
+                0.028089376449584962,
+                0.02814460754394531,
+                0.02809753608703613,
+                0.028185600280761718,
+                0.028219392776489258,
+                0.02817433547973633,
+                0.028188703536987304,
+                0.028273632049560547,
+                0.028203008651733398,
+                0.028272640228271483,
+                0.02832076835632324,
+                0.028286975860595705,
+                0.028462080001831053,
+                0.028271615982055662,
+                0.028267520904541016,
+                0.02813542366027832,
+                0.028083200454711913,
+                0.02817433547973633,
+                0.02813132858276367,
+                0.028169216156005858,
+                0.02814156723022461,
+                0.02831667137145996,
+                0.028165119171142578,
+                0.028230655670166017,
+                0.028100608825683594,
+                0.028112895965576173,
+                0.02818252754211426,
+                0.028258304595947265,
+                0.02819584083557129,
+                0.028207103729248048,
+                0.028217344284057616,
+                0.02815488052368164,
+                0.028200960159301756,
+                0.02815897560119629,
+                0.028227584838867188,
+                0.028274688720703125,
+                0.028177408218383788,
+                0.028203008651733398,
+                0.02821855926513672,
+                0.028306240081787108,
+                0.028197887420654297,
+                0.028224512100219725,
+                0.028236799240112305,
+                0.028216320037841795,
+                0.02862387275695801,
+                0.028222463607788087,
+                0.028224512100219725,
+                0.02840575981140137,
+                0.028248064041137694,
+                0.028196863174438477,
+                0.02814668846130371,
+                0.0281343994140625,
+                0.028056575775146485,
+                0.028081151962280275,
+                0.02815283203125,
+                0.02815795135498047,
+                0.02813337516784668,
+                0.02815999984741211,
+                0.0281395206451416,
+                0.02815795135498047,
+                0.028074047088623048,
+                0.028581823348999023,
+                0.028071935653686524,
+                0.028057600021362306,
+                0.027998207092285156,
+                0.028102655410766602,
+                0.028043264389038085,
+                0.02810572814941406,
+                0.028116992950439453,
+                0.02813849639892578,
+                0.02798080062866211,
+                0.028047359466552735,
+                0.0279736328125,
+                0.02817638397216797,
+                0.028185600280761718,
+                0.02818662452697754,
+                0.02817638397216797,
+                0.028223487854003908,
+                0.02813849639892578,
+                0.028263423919677736,
+                0.028211200714111328,
+                0.028252159118652344,
+                0.02816307258605957,
+                0.028212223052978515,
+                0.0285665283203125,
+                0.028265472412109374,
+                0.02815692710876465,
+                0.028211200714111328,
+                0.028277759552001954,
+                0.028258304595947265,
+                0.028265472412109374,
+                0.028255231857299806,
+                0.028119136810302734,
+                0.028214176177978514,
+                0.028092416763305664,
+                0.028078079223632812,
+                0.028099584579467773,
+                0.02812518310546875,
+                0.02818355178833008,
+                0.02814259147644043,
+                0.02812518310546875,
+                0.028203008651733398,
+                0.02836479949951172,
+                0.02810163116455078,
+                0.02812928009033203,
+                0.02819174385070801,
+                0.02814361572265625,
+                0.028270591735839845,
+                0.02812313652038574,
+                0.02814771270751953,
+                0.02820812797546387,
+                0.028250112533569335,
+                0.028112895965576173,
+                0.028212223052978515,
+                0.028161024093627928,
+                0.028227584838867188,
+                0.028719104766845704,
+                0.028396543502807618,
+                0.028242944717407226,
+                0.028296192169189452,
+                0.028049407958984376,
+                0.028107776641845703,
+                0.02796236801147461,
+                0.028043264389038085,
+                0.028154943466186525,
+                0.02812614440917969,
+                0.028103679656982423,
+                0.02835251235961914,
+                0.02819071960449219,
+                0.028238847732543947,
+                0.02814793586730957,
+                0.02830723190307617,
+                0.028259328842163086,
+                0.028862464904785157,
+                0.028201984405517577,
+                0.028250112533569335,
+                0.02815692710876465,
+                0.028301311492919923,
+                0.028246015548706056,
+                0.028302335739135744,
+                0.028301536560058595,
+                0.028325664520263673,
+                0.028278783798217775,
+                0.028412927627563478,
+                0.02832076835632324,
+                0.028400640487670898,
+                0.028233728408813476,
+                0.02836992073059082,
+                0.028254207611083985,
+                0.02835763168334961,
+                0.02814771270751953,
+                0.02815795135498047,
+                0.028282880783081055,
+                0.028290048599243164,
+                0.028263423919677736,
+                0.028271615982055662,
+                0.028233728408813476,
+                0.028288000106811522,
+                0.028236799240112305,
+                0.028411903381347657,
+                0.028379135131835938,
+                0.028399616241455077,
+                0.028203008651733398,
+                0.02833919906616211,
+                0.02832691192626953,
+                0.028264448165893553,
+                0.028293119430541993,
+                0.028215295791625978,
+                0.028255231857299806,
+                0.028318719863891603,
+                0.028249088287353515,
+                0.02834943962097168,
+                0.02832486343383789,
+                0.02853273582458496,
+                0.02834534454345703,
+                0.028299264907836914,
+                0.028269567489624024,
+                0.028277759552001954,
+                0.02836992073059082,
+                0.028308479309082032,
+                0.028293119430541993,
+                0.028290048599243164,
+                0.02824608039855957,
+                0.028138431549072265,
+                0.028049407958984376,
+                0.02816819190979004,
+                0.02815692710876465,
+                0.02853785514831543,
+                0.028103679656982423,
+                0.028387327194213868,
+                0.028417024612426758,
+                0.02836787223815918,
+                0.028466175079345703,
+                0.028445695877075194,
+                0.028523519515991212,
+                0.028376256942749024,
+                0.028446527481079103,
+                0.028334079742431642,
+                0.028413951873779295,
+                0.028468223571777345,
+                0.028483583450317384,
+                0.02835148811340332,
+                0.02854604721069336,
+                0.028906496047973632,
+                0.028411903381347657,
+                0.028536832809448243,
+                0.028470272064208983,
+                0.028387327194213868,
+                0.02838835144042969,
+                0.028246015548706056,
+                0.028404735565185548,
+                0.028262399673461915,
+                0.02834124755859375,
+                0.02837196731567383,
+                0.02851020812988281,
+                0.028463104248046874,
+                0.028523519515991212,
+                0.028493824005126952,
+                0.028448768615722656,
+                0.028403711318969727,
+                0.02852659225463867,
+                0.028461055755615236,
+                0.02840575981140137,
+                0.02832486343383789,
+                0.02831974411010742,
+                0.028483583450317384,
+                0.02842624092102051,
+                0.028267520904541016,
+                0.028464128494262695,
+                0.02840166473388672,
+                0.028398591995239256,
+                0.028468223571777345,
+                0.028309503555297853,
+                0.02837299156188965,
+                0.028447744369506835,
+                0.02853376007080078,
+                0.028483583450317384,
+                0.02838937568664551,
+                0.028440576553344726,
+                0.028499967575073244,
+                0.028471296310424804,
+                0.028471296310424804,
+                0.02856857681274414,
+                0.028427263259887696,
+                0.02835251235961914,
+                0.028405855178833008,
+                0.028382112503051758,
+                0.02840575981140137,
+                0.02835558319091797,
+                0.02834739112854004,
+                0.02835763168334961,
+                0.02837196731567383,
+                0.028438528060913085,
+                0.028485631942749022,
+                0.02854092788696289,
+                0.02839449691772461,
+                0.028420095443725587,
+                0.028702720642089844,
+                0.028379135131835938,
+                0.02837606430053711,
+                0.028411903381347657,
+                0.028253183364868165,
+                0.02832793617248535,
+                0.028269567489624024,
+                0.028448768615722656,
+                0.028379135131835938,
+                0.028407808303833007,
+                0.028234752655029297,
+                0.02836070442199707,
+                0.028237823486328126,
+                0.028314624786376953,
+                0.028256256103515624,
+                0.028459007263183594,
+                0.028318719863891603,
+                0.02896691131591797,
+                0.02837606430053711,
+                0.028451839447021485,
+                0.02839756774902344,
+                0.028398591995239256,
+                0.028196863174438477,
+                0.028280832290649413,
+                0.02815283203125,
+                0.028207103729248048,
+                0.028235776901245117,
+                0.02840985679626465,
+                0.02819891166687012,
+                0.028259328842163086,
+                0.02813542366027832,
+                0.028144639968872072,
+                0.02816819190979004,
+                0.028212223052978515,
+                0.02815488052368164,
+                0.028284927368164063,
+                0.028212223052978515,
+                0.02832076835632324,
+                0.028273664474487304,
+                0.028285951614379884,
+                0.028275711059570312,
+                0.02833612823486328,
+                0.028241920471191406,
+                0.028375040054321288,
+                0.028286975860595705,
+                0.02834432029724121,
+                0.028305408477783203,
+                0.028256256103515624,
+                0.028310527801513673,
+                0.028410879135131836,
+                0.02834230422973633,
+                0.02840982437133789,
+                0.028298240661621094,
+                0.028252159118652344,
+                0.028215295791625978,
+                0.02833305549621582,
+                0.028300287246704102,
+                0.028396543502807618,
+                0.028319839477539063,
+                0.0283155517578125,
+                0.028309503555297853,
+                0.02834943962097168,
+                0.02834124755859375,
+                0.028387327194213868,
+                0.028267520904541016,
+                0.028297216415405273,
+                0.028283903121948242,
+                0.028301311492919923,
+                0.028329984664916992,
+                0.028281856536865234,
+                0.028274688720703125,
+                0.02873855972290039,
+                0.028413951873779295,
+                0.02839756774902344,
+                0.028281856536865234,
+                0.028215295791625978,
+                0.02836992073059082,
+                0.02833510398864746,
+                0.028289024353027343,
+                0.02857369613647461,
+                0.03272192001342773,
+                0.033925121307373046,
+                0.033454078674316406,
+                0.032860160827636715,
+                0.03234201431274414,
+                0.032118785858154295,
+                0.03211673736572265,
+                0.031765504837036135,
+                0.0319866886138916,
+                0.03199692726135254,
+                0.032059391021728514,
+                0.031221759796142577,
+                0.03174297523498535,
+                0.02983628845214844,
+                0.031410175323486327,
+                0.03156991958618164,
+                0.03136614418029785,
+                0.029371391296386717,
+                0.029903104782104492,
+                0.028296960830688476,
+                0.028291072845458985,
+                0.028262399673461915,
+                0.028241920471191406,
+                0.028240896224975585,
+                0.02835456085205078,
+                0.028322816848754883,
+                0.028276735305786133,
+                0.028301311492919923,
+                0.028325887680053712,
+                0.02839347267150879,
+                0.02836992073059082,
+                0.028276735305786133,
+                0.028267520904541016,
+                0.02838937568664551,
+                0.028480512619018555,
+                0.02832486343383789,
+                0.028279008865356444,
+                0.02837481689453125,
+                0.028370943069458008,
+                0.02835456085205078,
+                0.02836582374572754,
+                0.02833510398864746,
+                0.028313600540161132,
+                0.028338176727294922,
+                0.02830745506286621,
+                0.028270591735839845,
+                0.028216320037841795,
+                0.028236799240112305,
+                0.028305408477783203,
+                0.028454912185668944,
+                0.028297216415405273,
+                0.028229631423950196,
+                0.028177408218383788,
+                0.028520448684692383,
+                0.028283903121948242,
+                0.028456960678100586,
+                0.02815180778503418,
+                0.028234752655029297,
+                0.02810163116455078,
+                0.028314624786376953,
+                0.02857062339782715,
+                0.028265472412109374,
+                0.028256256103515624,
+                0.02836479949951172,
+                0.02835353660583496,
+                0.02836992073059082,
+                0.028278783798217775,
+                0.02835251235961914,
+                0.0281395206451416,
+                0.028673023223876954,
+                0.028110847473144532,
+                0.02831667137145996,
+                0.028214271545410157,
+                0.028331008911132813,
+                0.028264448165893553,
+                0.02840985679626465,
+                0.028246015548706056,
+                0.02872319984436035,
+                0.028240896224975585,
+                0.028302335739135744,
+                0.028257280349731444,
+                0.028492799758911135,
+                0.028256320953369142,
+                0.028370880126953126,
+                0.028271615982055662,
+                0.028445695877075194,
+                0.028283903121948242,
+                0.028247039794921876,
+                0.02830335998535156,
+                0.028494848251342773,
+                0.02838425636291504,
+                0.028338176727294922,
+                0.028227807998657227,
+                0.028345119476318358,
+                0.028215295791625978,
+                0.028278783798217775,
+                0.02814668846130371,
+                0.028266496658325195,
+                0.028161024093627928,
+                0.028165119171142578,
+                0.02818252754211426,
+                0.028205055236816406,
+                0.027983871459960938,
+                0.02815180778503418,
+                0.028107776641845703,
+                0.028302335739135744,
+                0.02817638397216797
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 35.178374330310795
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}