Upload /gemma-7b/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json with huggingface_hub

gemma-7b/nf4_batch_size_1_sq_len_256_new_tokens_256/benchmark_report.json

@@ -0,0 +1,755 @@
+{
+    "prefill": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1092.5056,
+            "max_vram": 6796.869632,
+            "max_reserved": 6312.42752,
+            "max_allocated": 6191.16288
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 0.0554454770588743,
+            "stdev": 0.0004663816241365346,
+            "values": [
+                0.06086252975463867,
+                0.057556991577148435,
+                0.05733478546142578,
+                0.05619302368164063,
+                0.055826431274414064,
+                0.05585203170776367,
+                0.05542195129394531,
+                0.05526425552368164,
+                0.055316478729248046,
+                0.0552806396484375,
+                0.05524991989135742,
+                0.05527142333984375,
+                0.05532160186767578,
+                0.05532364654541016,
+                0.05526732635498047,
+                0.0552540168762207,
+                0.05535948944091797,
+                0.05530214309692383,
+                0.05532262420654297,
+                0.05526732635498047,
+                0.05538508987426758,
+                0.05527865600585938,
+                0.055395328521728515,
+                0.05530112075805664,
+                0.055370880126953126,
+                0.05529395294189453,
+                0.055354366302490236,
+                0.05530419158935547,
+                0.055410785675048826,
+                0.055329887390136716,
+                0.05536767959594727,
+                0.05533497619628906,
+                0.05533798217773438,
+                0.05532793426513672,
+                0.05535027313232422,
+                0.05534515380859375,
+                0.055346176147460936,
+                0.055387134552001956,
+                0.05535334396362305,
+                0.05540556716918945,
+                0.05538816070556641,
+                0.05535129547119141,
+                0.05537484741210937,
+                0.05542195129394531,
+                0.05540454483032226,
+                0.05540249633789063,
+                0.05536972808837891,
+                0.055444480895996094,
+                0.055378944396972656,
+                0.055462913513183595,
+                0.055411712646484375,
+                0.05542195129394531,
+                0.055416831970214846,
+                0.05541388702392578,
+                0.055414783477783204,
+                0.055430145263671876,
+                0.05539126586914062,
+                0.05540454483032226,
+                0.05533184051513672,
+                0.05538918304443359,
+                0.05534822463989258,
+                0.05540153503417969,
+                0.05540761566162109,
+                0.05541888046264649,
+                0.05538508987426758,
+                0.05533695983886719,
+                0.05538816070556641,
+                0.05535232162475586,
+                0.055390209197998044,
+                0.05537484741210937,
+                0.05534003067016602,
+                0.05540966415405273,
+                0.05534822463989258,
+                0.05533388900756836,
+                0.055419902801513675,
+                0.05529203033447266,
+                0.05536153411865234,
+                0.055375873565673826,
+                0.055392257690429686,
+                0.05540569686889649,
+                0.055400447845458986,
+                0.055406593322753904,
+                0.05535027313232422,
+                0.05541580963134766,
+                0.055362560272216796,
+                0.055406593322753904,
+                0.05544243240356445,
+                0.05542502212524414,
+                0.05542092895507812,
+                0.05539126586914062,
+                0.05543423843383789,
+                0.05539123153686523,
+                0.05543526458740235,
+                0.05542092895507812,
+                0.055403518676757815,
+                0.05536870574951172,
+                0.055387134552001956,
+                0.05540556716918945,
+                0.05535232162475586,
+                0.05537996673583984,
+                0.05544460678100586,
+                0.05541273498535156,
+                0.05544140625,
+                0.05540780639648438,
+                0.05541580963134766,
+                0.0553994255065918,
+                0.05532057571411133,
+                0.05541273498535156,
+                0.05542195129394531,
+                0.055392513275146486,
+                0.05536972808837891,
+                0.05535027313232422,
+                0.0553994255065918,
+                0.05535539245605469,
+                0.055411712646484375,
+                0.05534342575073242,
+                0.055393280029296874,
+                0.055346176147460936,
+                0.05536665725708008,
+                0.055431167602539064,
+                0.05539737701416016,
+                0.055354366302490236,
+                0.055375873565673826,
+                0.055362560272216796,
+                0.055382015228271485,
+                0.05544038391113281,
+                0.055413761138916016,
+                0.05538918304443359,
+                0.05534310531616211,
+                0.05535846328735351,
+                0.05537177658081055,
+                0.05534310531616211,
+                0.055662593841552734,
+                0.05537177658081055,
+                0.05538918304443359,
+                0.055300094604492187,
+                0.055373825073242185,
+                0.055400447845458986,
+                0.0555335693359375,
+                0.055416831970214846,
+                0.05536153411865234,
+                0.05539737701416016,
+                0.05540556716918945,
+                0.05542604827880859,
+                0.05539228820800781,
+                0.055384063720703126,
+                0.05541273498535156,
+                0.05540985488891602,
+                0.05543244934082031,
+                0.05541888046264649,
+                0.055416831970214846,
+                0.05532979202270508,
+                0.055419166564941405,
+                0.05542502212524414,
+                0.05542604827880859,
+                0.055446529388427736,
+                0.055408641815185546,
+                0.05539430236816406,
+                0.055387134552001956,
+                0.055465984344482425,
+                0.05535129547119141,
+                0.05542412948608398,
+                0.05538304138183594,
+                0.05541888046264649,
+                0.0553994255065918,
+                0.05537484741210937,
+                0.05544345474243164,
+                0.0553994255065918,
+                0.055354366302490236,
+                0.0553809928894043,
+                0.05542092895507812,
+                0.055363582611083983,
+                0.055454814910888675,
+                0.055408641815185546,
+                0.05529395294189453,
+                0.055400447845458986,
+                0.0553994255065918,
+                0.05542915344238281,
+                0.05540249633789063,
+                0.05542707061767578,
+                0.05538524627685547
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 4617.14847774091
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "decode": {
+        "memory": {
+            "unit": "MB",
+            "max_ram": 1092.5056,
+            "max_vram": 7212.105728,
+            "max_reserved": 6727.663616,
+            "max_allocated": 6570.75456
+        },
+        "latency": {
+            "unit": "s",
+            "mean": 7.895504375457767,
+            "stdev": 0.03317504310608088,
+            "values": [
+                7.928679418563847,
+                7.862329332351686
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 32.2968600704772
+        },
+        "energy": null,
+        "efficiency": null
+    },
+    "per_token": {
+        "memory": null,
+        "latency": {
+            "unit": "s",
+            "mean": 0.03096276225669712,
+            "stdev": 0.0007418630006546437,
+            "values": [
+                0.03081625556945801,
+                0.031161344528198243,
+                0.030646272659301758,
+                0.03083673667907715,
+                0.030887935638427736,
+                0.030846975326538087,
+                0.03084492874145508,
+                0.030793727874755858,
+                0.03078246307373047,
+                0.03079475212097168,
+                0.030785535812377928,
+                0.030809087753295897,
+                0.030712831497192384,
+                0.03077017593383789,
+                0.03467468643188477,
+                0.035454975128173825,
+                0.03563008117675781,
+                0.03514572906494141,
+                0.0348487663269043,
+                0.03461939239501953,
+                0.03455692672729492,
+                0.03460710525512695,
+                0.03475251388549805,
+                0.034625537872314455,
+                0.03277926254272461,
+                0.032715774536132815,
+                0.03133132743835449,
+                0.033539070129394534,
+                0.032826465606689455,
+                0.03347241592407227,
+                0.032421886444091795,
+                0.031038463592529295,
+                0.03073023986816406,
+                0.030729215621948244,
+                0.030690303802490236,
+                0.030704639434814454,
+                0.0307957763671875,
+                0.030838783264160157,
+                0.030740480422973632,
+                0.030740480422973632,
+                0.03074764823913574,
+                0.030766080856323243,
+                0.03078348731994629,
+                0.03077631950378418,
+                0.03134668731689453,
+                0.031099903106689454,
+                0.03076198387145996,
+                0.030817279815673827,
+                0.030736383438110353,
+                0.031029247283935548,
+                0.030942207336425782,
+                0.03074662399291992,
+                0.030700544357299804,
+                0.030785535812377928,
+                0.030657535552978517,
+                0.030769151687622072,
+                0.030698495864868162,
+                0.030673919677734376,
+                0.03076812744140625,
+                0.03081625556945801,
+                0.03083263969421387,
+                0.03079270362854004,
+                0.030847999572753908,
+                0.03072204780578613,
+                0.030718975067138672,
+                0.03119001579284668,
+                0.03085312080383301,
+                0.03077324867248535,
+                0.03082444763183594,
+                0.030843904495239258,
+                0.03078041648864746,
+                0.030716928482055664,
+                0.03076198387145996,
+                0.030907392501831055,
+                0.030675968170166015,
+                0.030741504669189453,
+                0.030874624252319335,
+                0.031073280334472656,
+                0.030836832046508788,
+                0.030646175384521485,
+                0.030741504669189453,
+                0.030703615188598633,
+                0.030765056610107422,
+                0.030855167388916017,
+                0.030675968170166015,
+                0.03079884719848633,
+                0.030819328308105468,
+                0.03077222442626953,
+                0.030721023559570314,
+                0.030728191375732423,
+                0.030712831497192384,
+                0.03077120018005371,
+                0.030698495864868162,
+                0.030657535552978517,
+                0.030687231063842774,
+                0.030693376541137695,
+                0.030731391906738282,
+                0.0306932487487793,
+                0.030877695083618165,
+                0.030657535552978517,
+                0.030698495864868162,
+                0.03082342338562012,
+                0.03077939224243164,
+                0.030850048065185546,
+                0.030744575500488282,
+                0.030687231063842774,
+                0.030749696731567383,
+                0.030879743576049806,
+                0.03078758430480957,
+                0.03075481605529785,
+                0.030676992416381835,
+                0.03072204780578613,
+                0.030850048065185546,
+                0.030676992416381835,
+                0.030814207077026368,
+                0.03122790336608887,
+                0.030826496124267577,
+                0.030756864547729492,
+                0.030823488235473633,
+                0.030783424377441405,
+                0.030742528915405274,
+                0.03079987144470215,
+                0.03079987144470215,
+                0.030901248931884766,
+                0.030727167129516602,
+                0.03080601692199707,
+                0.030846975326538087,
+                0.030818304061889647,
+                0.030757888793945313,
+                0.03078758430480957,
+                0.03100876808166504,
+                0.031054847717285155,
+                0.03077120018005371,
+                0.03081318473815918,
+                0.030737407684326173,
+                0.030785535812377928,
+                0.030838783264160157,
+                0.03077631950378418,
+                0.03715584182739258,
+                0.035434497833251956,
+                0.03479449462890625,
+                0.03339571380615235,
+                0.031710208892822264,
+                0.030769151687622072,
+                0.030708736419677734,
+                0.030838783264160157,
+                0.03073244857788086,
+                0.030737247467041016,
+                0.03075993537902832,
+                0.03075481605529785,
+                0.030882816314697265,
+                0.0307589111328125,
+                0.030900224685668946,
+                0.030785535812377928,
+                0.03082444763183594,
+                0.030835712432861328,
+                0.03077017593383789,
+                0.030830591201782227,
+                0.030880767822265624,
+                0.030858240127563476,
+                0.03077836799621582,
+                0.03082137680053711,
+                0.030860288619995117,
+                0.030826496124267577,
+                0.030736383438110353,
+                0.03080294418334961,
+                0.030740480422973632,
+                0.030834688186645507,
+                0.030833663940429686,
+                0.03079680061340332,
+                0.03081625556945801,
+                0.03077222442626953,
+                0.03082444763183594,
+                0.030745599746704103,
+                0.03078665542602539,
+                0.030721952438354492,
+                0.030744575500488282,
+                0.030710784912109376,
+                0.030850048065185546,
+                0.030749696731567383,
+                0.03125760078430176,
+                0.03080499267578125,
+                0.032191680908203124,
+                0.0311662712097168,
+                0.030852096557617188,
+                0.03080601692199707,
+                0.030819328308105468,
+                0.03077222442626953,
+                0.030774272918701173,
+                0.03074662399291992,
+                0.03076812744140625,
+                0.030708736419677734,
+                0.03078348731994629,
+                0.03077120018005371,
+                0.03078246307373047,
+                0.031164415359497072,
+                0.030726144790649414,
+                0.030769151687622072,
+                0.030845951080322266,
+                0.03079987144470215,
+                0.03061759948730469,
+                0.03062579154968262,
+                0.030766080856323243,
+                0.030656511306762696,
+                0.030877695083618165,
+                0.030880767822265624,
+                0.030741600036621092,
+                0.03079465675354004,
+                0.030851072311401367,
+                0.030839807510375978,
+                0.030701568603515625,
+                0.03078246307373047,
+                0.03073244857788086,
+                0.03083247947692871,
+                0.03093926429748535,
+                0.030780288696289064,
+                0.030809087753295897,
+                0.030760959625244142,
+                0.03082854461669922,
+                0.030793727874755858,
+                0.030810111999511718,
+                0.030852096557617188,
+                0.03077017593383789,
+                0.03077222442626953,
+                0.030911487579345705,
+                0.030766080856323243,
+                0.03080499267578125,
+                0.030711807250976563,
+                0.030811168670654296,
+                0.030770143508911132,
+                0.030753952026367187,
+                0.030772064208984377,
+                0.030732288360595703,
+                0.030712831497192384,
+                0.03079270362854004,
+                0.030878719329833985,
+                0.030817279815673827,
+                0.0307957763671875,
+                0.030712831497192384,
+                0.030740480422973632,
+                0.030895103454589845,
+                0.030740480422973632,
+                0.030841856002807616,
+                0.030765056610107422,
+                0.030818304061889647,
+                0.03121664047241211,
+                0.030819328308105468,
+                0.030715904235839843,
+                0.030817279815673827,
+                0.03079167938232422,
+                0.03074355125427246,
+                0.030868480682373047,
+                0.03077939224243164,
+                0.03079475212097168,
+                0.03080089569091797,
+                0.03084492874145508,
+                0.030757888793945313,
+                0.03078963279724121,
+                0.03078860855102539,
+                0.031169536590576172,
+                0.030860288619995117,
+                0.030740480422973632,
+                0.030822399139404297,
+                0.03075584030151367,
+                0.030855167388916017,
+                0.030705663681030275,
+                0.030954496383666992,
+                0.03075174331665039,
+                0.0307640323638916,
+                0.030744575500488282,
+                0.030860288619995117,
+                0.03080703926086426,
+                0.03079884719848633,
+                0.030749696731567383,
+                0.03078348731994629,
+                0.030732288360595703,
+                0.030903295516967775,
+                0.030689279556274415,
+                0.03079884719848633,
+                0.03075993537902832,
+                0.030895103454589845,
+                0.030865407943725585,
+                0.030849023818969725,
+                0.0307589111328125,
+                0.030850048065185546,
+                0.03077120018005371,
+                0.03079167938232422,
+                0.030729215621948244,
+                0.030857215881347655,
+                0.03082444763183594,
+                0.030873600006103515,
+                0.030769151687622072,
+                0.030765056610107422,
+                0.030684160232543944,
+                0.030870527267456056,
+                0.03076300811767578,
+                0.03081625556945801,
+                0.03077939224243164,
+                0.030769151687622072,
+                0.030827520370483398,
+                0.030879743576049806,
+                0.030699520111083983,
+                0.030873600006103515,
+                0.030873600006103515,
+                0.030930944442749023,
+                0.03074662399291992,
+                0.03077529525756836,
+                0.030870527267456056,
+                0.0307589111328125,
+                0.03132825660705566,
+                0.03078758430480957,
+                0.0307640323638916,
+                0.03077836799621582,
+                0.031096832275390625,
+                0.03077939224243164,
+                0.03077836799621582,
+                0.03080499267578125,
+                0.030863359451293947,
+                0.030882816314697265,
+                0.030911487579345705,
+                0.030827520370483398,
+                0.030704639434814454,
+                0.03079680061340332,
+                0.03082444763183594,
+                0.03127910423278808,
+                0.03076812744140625,
+                0.030865407943725585,
+                0.030745599746704103,
+                0.03082956886291504,
+                0.03079270362854004,
+                0.030860288619995117,
+                0.03073843193054199,
+                0.03079475212097168,
+                0.030833663940429686,
+                0.030826496124267577,
+                0.030744575500488282,
+                0.030737407684326173,
+                0.030678016662597656,
+                0.030865407943725585,
+                0.03075481605529785,
+                0.03075993537902832,
+                0.03094118309020996,
+                0.030649343490600587,
+                0.03078348731994629,
+                0.030849023818969725,
+                0.03095756721496582,
+                0.030733312606811523,
+                0.03078144073486328,
+                0.030864383697509764,
+                0.030793727874755858,
+                0.03075071907043457,
+                0.030826496124267577,
+                0.03073843193054199,
+                0.030728191375732423,
+                0.030857215881347655,
+                0.03082956886291504,
+                0.030809087753295897,
+                0.03073023986816406,
+                0.030943231582641603,
+                0.030929920196533203,
+                0.030892032623291016,
+                0.030856191635131838,
+                0.030845951080322266,
+                0.030867456436157226,
+                0.030856191635131838,
+                0.03080703926086426,
+                0.0307906551361084,
+                0.03079270362854004,
+                0.030810111999511718,
+                0.03081216049194336,
+                0.030879743576049806,
+                0.030850048065185546,
+                0.030882816314697265,
+                0.03085312080383301,
+                0.031268863677978515,
+                0.030932992935180665,
+                0.030842880249023437,
+                0.03079475212097168,
+                0.030857215881347655,
+                0.030814207077026368,
+                0.030883840560913086,
+                0.03078144073486328,
+                0.03077017593383789,
+                0.030869504928588868,
+                0.030830591201782227,
+                0.03077017593383789,
+                0.030895103454589845,
+                0.03080396842956543,
+                0.03079782485961914,
+                0.030867456436157226,
+                0.031406080245971676,
+                0.030826496124267577,
+                0.030826496124267577,
+                0.030801919937133788,
+                0.030830591201782227,
+                0.03093708801269531,
+                0.03073945617675781,
+                0.03082956886291504,
+                0.030864383697509764,
+                0.030818304061889647,
+                0.030830591201782227,
+                0.030710784912109376,
+                0.030757888793945313,
+                0.030861312866210938,
+                0.030801919937133788,
+                0.030862335205078126,
+                0.030756864547729492,
+                0.03076300811767578,
+                0.03079782485961914,
+                0.030866432189941406,
+                0.031041536331176758,
+                0.03082342338562012,
+                0.030741504669189453,
+                0.03080806350708008,
+                0.030833663940429686,
+                0.030908416748046875,
+                0.03074764823913574,
+                0.030870527267456056,
+                0.03078758430480957,
+                0.03080294418334961,
+                0.030748672485351562,
+                0.030680063247680665,
+                0.030690303802490236,
+                0.03077734375,
+                0.030736383438110353,
+                0.03082854461669922,
+                0.030716928482055664,
+                0.03084492874145508,
+                0.030932992935180665,
+                0.030822399139404297,
+                0.030934015274047853,
+                0.030841856002807616,
+                0.030839807510375978,
+                0.030843904495239258,
+                0.03080703926086426,
+                0.030983167648315428,
+                0.030835712432861328,
+                0.030757888793945313,
+                0.030887935638427736,
+                0.031033344268798828,
+                0.03079270362854004,
+                0.03082342338562012,
+                0.03081318473815918,
+                0.03084492874145508,
+                0.03078451156616211,
+                0.030814207077026368,
+                0.03081625556945801,
+                0.030729215621948244,
+                0.030866432189941406,
+                0.030885887145996094,
+                0.030685184478759765,
+                0.03096575927734375,
+                0.03083263969421387,
+                0.030883840560913086,
+                0.030912511825561522,
+                0.0310118408203125,
+                0.030694400787353516,
+                0.03077939224243164,
+                0.030895103454589845,
+                0.03079987144470215,
+                0.030879743576049806,
+                0.03073023986816406,
+                0.03076812744140625,
+                0.030737407684326173,
+                0.030946304321289062,
+                0.03079270362854004,
+                0.03079167938232422,
+                0.03072928047180176,
+                0.030773183822631837,
+                0.030817279815673827,
+                0.03077631950378418,
+                0.03072204780578613,
+                0.03082342338562012,
+                0.030838783264160157,
+                0.03098111915588379,
+                0.03082035255432129,
+                0.03080703926086426,
+                0.030739551544189454,
+                0.03076291275024414,
+                0.030855167388916017,
+                0.030793727874755858,
+                0.030733312606811523,
+                0.03077017593383789,
+                0.03081523132324219,
+                0.03080703926086426,
+                0.03077836799621582,
+                0.030658559799194338,
+                0.0324640007019043,
+                0.031488895416259766,
+                0.03078041648864746,
+                0.03075584030151367,
+                0.030697471618652345,
+                0.030693376541137695,
+                0.03078246307373047,
+                0.03075584030151367,
+                0.03080601692199707,
+                0.030621696472167968,
+                0.030688255310058594,
+                0.030740480422973632,
+                0.030710784912109376,
+                0.03080499267578125,
+                0.03076812744140625,
+                0.030735359191894532,
+                0.030817279815673827,
+                0.03128422355651855,
+                0.031218687057495118,
+                0.030834688186645507,
+                0.03079884719848633,
+                0.030842880249023437,
+                0.03077631950378418
+            ]
+        },
+        "throughput": {
+            "unit": "tokens/s",
+            "value": 32.29686007047721
+        },
+        "energy": null,
+        "efficiency": null
+    }
+}