| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 10.762331838565022, |
| "eval_steps": 500, |
| "global_step": 300, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.35874439461883406, |
| "grad_norm": 5.573123455047607, |
| "learning_rate": 9.259259259259259e-07, |
| "logits/chosen": -2.3576977252960205, |
| "logits/rejected": -2.3553850650787354, |
| "logps/chosen": -61.0846061706543, |
| "logps/rejected": -126.1152572631836, |
| "loss": 0.6942, |
| "rewards/accuracies": 0.36250001192092896, |
| "rewards/chosen": -0.0012381849810481071, |
| "rewards/margins": -0.010831715539097786, |
| "rewards/rejected": 0.009593529626727104, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.7174887892376681, |
| "grad_norm": 3.8175244331359863, |
| "learning_rate": 1.8518518518518519e-06, |
| "logits/chosen": -2.3618054389953613, |
| "logits/rejected": -2.384873867034912, |
| "logps/chosen": -63.2865104675293, |
| "logps/rejected": -81.8724365234375, |
| "loss": 0.6925, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -0.002874236088246107, |
| "rewards/margins": -0.003056168556213379, |
| "rewards/rejected": 0.00018193255527876318, |
| "step": 20 |
| }, |
| { |
| "epoch": 1.0762331838565022, |
| "grad_norm": 4.383662700653076, |
| "learning_rate": 2.7777777777777783e-06, |
| "logits/chosen": -2.3333091735839844, |
| "logits/rejected": -2.3545360565185547, |
| "logps/chosen": -76.02193450927734, |
| "logps/rejected": -88.19218444824219, |
| "loss": 0.6932, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.008687756024301052, |
| "rewards/margins": -0.0016246589366346598, |
| "rewards/rejected": -0.007063096854835749, |
| "step": 30 |
| }, |
| { |
| "epoch": 1.4349775784753362, |
| "grad_norm": 4.095139503479004, |
| "learning_rate": 3.7037037037037037e-06, |
| "logits/chosen": -2.3483197689056396, |
| "logits/rejected": -2.363131523132324, |
| "logps/chosen": -66.08761596679688, |
| "logps/rejected": -78.22222900390625, |
| "loss": 0.6889, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.014177674427628517, |
| "rewards/margins": 0.012472175993025303, |
| "rewards/rejected": -0.026649847626686096, |
| "step": 40 |
| }, |
| { |
| "epoch": 1.7937219730941703, |
| "grad_norm": 4.034746170043945, |
| "learning_rate": 4.62962962962963e-06, |
| "logits/chosen": -2.31459903717041, |
| "logits/rejected": -2.335706949234009, |
| "logps/chosen": -69.31814575195312, |
| "logps/rejected": -77.91527557373047, |
| "loss": 0.6823, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -0.04294499754905701, |
| "rewards/margins": 0.007987136952579021, |
| "rewards/rejected": -0.050932131707668304, |
| "step": 50 |
| }, |
| { |
| "epoch": 2.1524663677130045, |
| "grad_norm": 4.620233535766602, |
| "learning_rate": 4.998119881260576e-06, |
| "logits/chosen": -2.364583969116211, |
| "logits/rejected": -2.3598217964172363, |
| "logps/chosen": -69.84207153320312, |
| "logps/rejected": -79.67817687988281, |
| "loss": 0.6717, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.06739393621683121, |
| "rewards/margins": 0.050740111619234085, |
| "rewards/rejected": -0.118134044110775, |
| "step": 60 |
| }, |
| { |
| "epoch": 2.5112107623318387, |
| "grad_norm": 4.450082302093506, |
| "learning_rate": 4.9866405060165044e-06, |
| "logits/chosen": -2.298379421234131, |
| "logits/rejected": -2.3169469833374023, |
| "logps/chosen": -79.13768005371094, |
| "logps/rejected": -79.24192810058594, |
| "loss": 0.6499, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.1282506138086319, |
| "rewards/margins": 0.08815944194793701, |
| "rewards/rejected": -0.21641004085540771, |
| "step": 70 |
| }, |
| { |
| "epoch": 2.8699551569506725, |
| "grad_norm": 4.7328877449035645, |
| "learning_rate": 4.964774158361991e-06, |
| "logits/chosen": -2.3363795280456543, |
| "logits/rejected": -2.3331849575042725, |
| "logps/chosen": -77.52214050292969, |
| "logps/rejected": -86.90077209472656, |
| "loss": 0.6346, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.22365322709083557, |
| "rewards/margins": 0.1937221735715866, |
| "rewards/rejected": -0.4173754155635834, |
| "step": 80 |
| }, |
| { |
| "epoch": 3.2286995515695067, |
| "grad_norm": 4.016219615936279, |
| "learning_rate": 4.93261217644956e-06, |
| "logits/chosen": -2.30871844291687, |
| "logits/rejected": -2.3049044609069824, |
| "logps/chosen": -65.6957778930664, |
| "logps/rejected": -77.02860260009766, |
| "loss": 0.6004, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.23905189335346222, |
| "rewards/margins": 0.2932591438293457, |
| "rewards/rejected": -0.5323110222816467, |
| "step": 90 |
| }, |
| { |
| "epoch": 3.587443946188341, |
| "grad_norm": 4.553714752197266, |
| "learning_rate": 4.8902889044347e-06, |
| "logits/chosen": -2.3068125247955322, |
| "logits/rejected": -2.294462203979492, |
| "logps/chosen": -85.46375274658203, |
| "logps/rejected": -79.72854614257812, |
| "loss": 0.5816, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.3506511151790619, |
| "rewards/margins": 0.2997886538505554, |
| "rewards/rejected": -0.6504397392272949, |
| "step": 100 |
| }, |
| { |
| "epoch": 3.9461883408071747, |
| "grad_norm": 4.804734230041504, |
| "learning_rate": 4.837981131305475e-06, |
| "logits/chosen": -2.3317933082580566, |
| "logits/rejected": -2.3493664264678955, |
| "logps/chosen": -68.45785522460938, |
| "logps/rejected": -88.28421020507812, |
| "loss": 0.5672, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.4280410706996918, |
| "rewards/margins": 0.3461475670337677, |
| "rewards/rejected": -0.7741886377334595, |
| "step": 110 |
| }, |
| { |
| "epoch": 4.304932735426009, |
| "grad_norm": 4.650998592376709, |
| "learning_rate": 4.775907352415367e-06, |
| "logits/chosen": -2.3039748668670654, |
| "logits/rejected": -2.301835775375366, |
| "logps/chosen": -76.26728820800781, |
| "logps/rejected": -87.93345642089844, |
| "loss": 0.5258, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.4848151206970215, |
| "rewards/margins": 0.6179562211036682, |
| "rewards/rejected": -1.102771282196045, |
| "step": 120 |
| }, |
| { |
| "epoch": 4.663677130044843, |
| "grad_norm": 4.17809534072876, |
| "learning_rate": 4.70432685680402e-06, |
| "logits/chosen": -2.299182653427124, |
| "logits/rejected": -2.291717290878296, |
| "logps/chosen": -77.27394104003906, |
| "logps/rejected": -76.96590423583984, |
| "loss": 0.5141, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.5012012720108032, |
| "rewards/margins": 0.39593780040740967, |
| "rewards/rejected": -0.8971391916275024, |
| "step": 130 |
| }, |
| { |
| "epoch": 5.022421524663677, |
| "grad_norm": 4.4158735275268555, |
| "learning_rate": 4.623538644118244e-06, |
| "logits/chosen": -2.267500400543213, |
| "logits/rejected": -2.271239995956421, |
| "logps/chosen": -77.53614807128906, |
| "logps/rejected": -105.53443908691406, |
| "loss": 0.4929, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.47719526290893555, |
| "rewards/margins": 0.7529892921447754, |
| "rewards/rejected": -1.230184555053711, |
| "step": 140 |
| }, |
| { |
| "epoch": 5.381165919282511, |
| "grad_norm": 4.5556640625, |
| "learning_rate": 4.533880175657419e-06, |
| "logits/chosen": -2.2397818565368652, |
| "logits/rejected": -2.2796878814697266, |
| "logps/chosen": -75.30777740478516, |
| "logps/rejected": -97.59407043457031, |
| "loss": 0.4545, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.3897199034690857, |
| "rewards/margins": 0.8326279520988464, |
| "rewards/rejected": -1.2223479747772217, |
| "step": 150 |
| }, |
| { |
| "epoch": 5.739910313901345, |
| "grad_norm": 5.067012786865234, |
| "learning_rate": 4.435725964760331e-06, |
| "logits/chosen": -2.284294843673706, |
| "logits/rejected": -2.264681339263916, |
| "logps/chosen": -87.51654052734375, |
| "logps/rejected": -82.73191833496094, |
| "loss": 0.4392, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.42664116621017456, |
| "rewards/margins": 0.7053943872451782, |
| "rewards/rejected": -1.132035493850708, |
| "step": 160 |
| }, |
| { |
| "epoch": 6.098654708520179, |
| "grad_norm": 4.425904273986816, |
| "learning_rate": 4.329486012421531e-06, |
| "logits/chosen": -2.296356201171875, |
| "logits/rejected": -2.2917532920837402, |
| "logps/chosen": -76.75779724121094, |
| "logps/rejected": -90.15029907226562, |
| "loss": 0.4153, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.5271193385124207, |
| "rewards/margins": 0.7608373761177063, |
| "rewards/rejected": -1.2879568338394165, |
| "step": 170 |
| }, |
| { |
| "epoch": 6.457399103139013, |
| "grad_norm": 4.75029182434082, |
| "learning_rate": 4.215604094671835e-06, |
| "logits/chosen": -2.2561912536621094, |
| "logits/rejected": -2.253948450088501, |
| "logps/chosen": -68.14691162109375, |
| "logps/rejected": -85.39097595214844, |
| "loss": 0.3785, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.5065070390701294, |
| "rewards/margins": 1.024714708328247, |
| "rewards/rejected": -1.531221628189087, |
| "step": 180 |
| }, |
| { |
| "epoch": 6.816143497757848, |
| "grad_norm": 5.048355579376221, |
| "learning_rate": 4.094555908876765e-06, |
| "logits/chosen": -2.323347568511963, |
| "logits/rejected": -2.284219741821289, |
| "logps/chosen": -74.8355484008789, |
| "logps/rejected": -85.9789047241211, |
| "loss": 0.3697, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.533603310585022, |
| "rewards/margins": 1.2120137214660645, |
| "rewards/rejected": -1.7456169128417969, |
| "step": 190 |
| }, |
| { |
| "epoch": 7.174887892376682, |
| "grad_norm": 5.01900053024292, |
| "learning_rate": 3.966847086696045e-06, |
| "logits/chosen": -2.300300121307373, |
| "logits/rejected": -2.3180909156799316, |
| "logps/chosen": -86.38873291015625, |
| "logps/rejected": -89.72538757324219, |
| "loss": 0.3611, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.7510987520217896, |
| "rewards/margins": 0.9809296727180481, |
| "rewards/rejected": -1.7320283651351929, |
| "step": 200 |
| }, |
| { |
| "epoch": 7.533632286995516, |
| "grad_norm": 4.92667818069458, |
| "learning_rate": 3.833011082004229e-06, |
| "logits/chosen": -2.297994375228882, |
| "logits/rejected": -2.288382053375244, |
| "logps/chosen": -71.64348602294922, |
| "logps/rejected": -84.01541900634766, |
| "loss": 0.3248, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.8383395075798035, |
| "rewards/margins": 1.047420859336853, |
| "rewards/rejected": -1.8857605457305908, |
| "step": 210 |
| }, |
| { |
| "epoch": 7.8923766816143495, |
| "grad_norm": 5.767988204956055, |
| "learning_rate": 3.693606942594873e-06, |
| "logits/chosen": -2.194643259048462, |
| "logits/rejected": -2.1788828372955322, |
| "logps/chosen": -83.70150756835938, |
| "logps/rejected": -101.75688171386719, |
| "loss": 0.3004, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.8993238210678101, |
| "rewards/margins": 1.521676778793335, |
| "rewards/rejected": -2.4210009574890137, |
| "step": 220 |
| }, |
| { |
| "epoch": 8.251121076233185, |
| "grad_norm": 4.738667964935303, |
| "learning_rate": 3.549216974976073e-06, |
| "logits/chosen": -2.2021820545196533, |
| "logits/rejected": -2.234846591949463, |
| "logps/chosen": -85.80267333984375, |
| "logps/rejected": -126.21333312988281, |
| "loss": 0.2709, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.1428090333938599, |
| "rewards/margins": 1.669283151626587, |
| "rewards/rejected": -2.8120923042297363, |
| "step": 230 |
| }, |
| { |
| "epoch": 8.609865470852018, |
| "grad_norm": 5.022470951080322, |
| "learning_rate": 3.400444312011776e-06, |
| "logits/chosen": -2.2339494228363037, |
| "logits/rejected": -2.259722948074341, |
| "logps/chosen": -79.16600036621094, |
| "logps/rejected": -100.4195327758789, |
| "loss": 0.2653, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.070847988128662, |
| "rewards/margins": 1.8333218097686768, |
| "rewards/rejected": -2.9041695594787598, |
| "step": 240 |
| }, |
| { |
| "epoch": 8.968609865470851, |
| "grad_norm": 6.2118096351623535, |
| "learning_rate": 3.2479103935691047e-06, |
| "logits/chosen": -2.197798490524292, |
| "logits/rejected": -2.2084178924560547, |
| "logps/chosen": -85.23640441894531, |
| "logps/rejected": -116.28971862792969, |
| "loss": 0.2426, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.2635414600372314, |
| "rewards/margins": 1.8116687536239624, |
| "rewards/rejected": -3.0752100944519043, |
| "step": 250 |
| }, |
| { |
| "epoch": 9.327354260089686, |
| "grad_norm": 5.798530101776123, |
| "learning_rate": 3.092252370695298e-06, |
| "logits/chosen": -2.1891565322875977, |
| "logits/rejected": -2.2006757259368896, |
| "logps/chosen": -77.15342712402344, |
| "logps/rejected": -129.3516845703125, |
| "loss": 0.2356, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.1600430011749268, |
| "rewards/margins": 2.2219691276550293, |
| "rewards/rejected": -3.382011890411377, |
| "step": 260 |
| }, |
| { |
| "epoch": 9.68609865470852, |
| "grad_norm": 6.839400768280029, |
| "learning_rate": 2.9341204441673267e-06, |
| "logits/chosen": -2.209928035736084, |
| "logits/rejected": -2.2113702297210693, |
| "logps/chosen": -79.22918701171875, |
| "logps/rejected": -105.42140197753906, |
| "loss": 0.2016, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.5536689758300781, |
| "rewards/margins": 1.9507955312728882, |
| "rewards/rejected": -3.504464626312256, |
| "step": 270 |
| }, |
| { |
| "epoch": 10.044843049327355, |
| "grad_norm": 4.958732604980469, |
| "learning_rate": 2.7741751485313295e-06, |
| "logits/chosen": -2.1307220458984375, |
| "logits/rejected": -2.1803646087646484, |
| "logps/chosen": -81.51334381103516, |
| "logps/rejected": -134.2724151611328, |
| "loss": 0.1909, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.3520208597183228, |
| "rewards/margins": 2.556453227996826, |
| "rewards/rejected": -3.9084744453430176, |
| "step": 280 |
| }, |
| { |
| "epoch": 10.403587443946188, |
| "grad_norm": 5.86897611618042, |
| "learning_rate": 2.6130845929767662e-06, |
| "logits/chosen": -2.139816999435425, |
| "logits/rejected": -2.14668607711792, |
| "logps/chosen": -87.5683822631836, |
| "logps/rejected": -116.4664535522461, |
| "loss": 0.1731, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.6710402965545654, |
| "rewards/margins": 2.187368869781494, |
| "rewards/rejected": -3.8584091663360596, |
| "step": 290 |
| }, |
| { |
| "epoch": 10.762331838565022, |
| "grad_norm": 4.753023147583008, |
| "learning_rate": 2.4515216705704396e-06, |
| "logits/chosen": -2.1322758197784424, |
| "logits/rejected": -2.1539082527160645, |
| "logps/chosen": -74.43647766113281, |
| "logps/rejected": -104.6937255859375, |
| "loss": 0.1715, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.6632559299468994, |
| "rewards/margins": 2.3588404655456543, |
| "rewards/rejected": -4.022096157073975, |
| "step": 300 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 540, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 20, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.1004136782823424e+18, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
