| { | |
| "best_metric": 1.3159173727035522, | |
| "best_model_checkpoint": "saves/Gemma-7B-It/lora/orpo-salt-half/checkpoint-1500", | |
| "epoch": 2.9974597798475866, | |
| "eval_steps": 500, | |
| "global_step": 1770, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01693480101608806, | |
| "grad_norm": 4.468957901000977, | |
| "learning_rate": 4.999614014035063e-06, | |
| "logits/chosen": 207.0987548828125, | |
| "logits/rejected": 208.2423858642578, | |
| "logps/chosen": -2.50309681892395, | |
| "logps/rejected": -2.976195812225342, | |
| "loss": 2.5989, | |
| "odds_ratio_loss": 0.9582594633102417, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.25030970573425293, | |
| "rewards/margins": 0.04730992019176483, | |
| "rewards/rejected": -0.29761961102485657, | |
| "sft_loss": 2.50309681892395, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.03386960203217612, | |
| "grad_norm": 3.255079984664917, | |
| "learning_rate": 4.998440543386042e-06, | |
| "logits/chosen": 207.7864227294922, | |
| "logits/rejected": 210.0211944580078, | |
| "logps/chosen": -2.684905529022217, | |
| "logps/rejected": -2.9186933040618896, | |
| "loss": 2.776, | |
| "odds_ratio_loss": 0.9109451174736023, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.26849058270454407, | |
| "rewards/margins": 0.023378772661089897, | |
| "rewards/rejected": -0.2918693423271179, | |
| "sft_loss": 2.684905529022217, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.05080440304826418, | |
| "grad_norm": 2.5176427364349365, | |
| "learning_rate": 4.996479918381253e-06, | |
| "logits/chosen": 212.6219024658203, | |
| "logits/rejected": 213.315673828125, | |
| "logps/chosen": -2.225409746170044, | |
| "logps/rejected": -2.3403306007385254, | |
| "loss": 2.3097, | |
| "odds_ratio_loss": 0.8430029153823853, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.22254100441932678, | |
| "rewards/margins": 0.01149209588766098, | |
| "rewards/rejected": -0.23403307795524597, | |
| "sft_loss": 2.225409746170044, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.06773920406435224, | |
| "grad_norm": 2.6412463188171387, | |
| "learning_rate": 4.993732756731818e-06, | |
| "logits/chosen": 212.7200164794922, | |
| "logits/rejected": 213.7319793701172, | |
| "logps/chosen": -2.2491908073425293, | |
| "logps/rejected": -2.6265273094177246, | |
| "loss": 2.3215, | |
| "odds_ratio_loss": 0.7235576510429382, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.22491908073425293, | |
| "rewards/margins": 0.03773364797234535, | |
| "rewards/rejected": -0.262652724981308, | |
| "sft_loss": 2.2491908073425293, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.0846740050804403, | |
| "grad_norm": 4.878486633300781, | |
| "learning_rate": 4.9901999239537345e-06, | |
| "logits/chosen": 218.13858032226562, | |
| "logits/rejected": 218.47933959960938, | |
| "logps/chosen": -2.325134754180908, | |
| "logps/rejected": -2.5971357822418213, | |
| "loss": 2.4052, | |
| "odds_ratio_loss": 0.8009692430496216, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.23251347243785858, | |
| "rewards/margins": 0.027200128883123398, | |
| "rewards/rejected": -0.2597135901451111, | |
| "sft_loss": 2.325134754180908, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.10160880609652836, | |
| "grad_norm": 1.934574842453003, | |
| "learning_rate": 4.985882533095186e-06, | |
| "logits/chosen": 218.4595184326172, | |
| "logits/rejected": 218.9486083984375, | |
| "logps/chosen": -2.1272051334381104, | |
| "logps/rejected": -2.2577621936798096, | |
| "loss": 2.2131, | |
| "odds_ratio_loss": 0.8588129878044128, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.21272051334381104, | |
| "rewards/margins": 0.013055694289505482, | |
| "rewards/rejected": -0.22577619552612305, | |
| "sft_loss": 2.1272051334381104, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.11854360711261643, | |
| "grad_norm": 8.395120620727539, | |
| "learning_rate": 4.9807819443858705e-06, | |
| "logits/chosen": 221.9539031982422, | |
| "logits/rejected": 223.3571319580078, | |
| "logps/chosen": -2.175661087036133, | |
| "logps/rejected": -2.3411145210266113, | |
| "loss": 2.2597, | |
| "odds_ratio_loss": 0.8399366140365601, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.2175661027431488, | |
| "rewards/margins": 0.016545329242944717, | |
| "rewards/rejected": -0.2341114580631256, | |
| "sft_loss": 2.175661087036133, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.1354784081287045, | |
| "grad_norm": 4.525905132293701, | |
| "learning_rate": 4.9748997648084404e-06, | |
| "logits/chosen": 224.2382354736328, | |
| "logits/rejected": 224.88278198242188, | |
| "logps/chosen": -1.8507779836654663, | |
| "logps/rejected": -2.099276304244995, | |
| "loss": 1.9368, | |
| "odds_ratio_loss": 0.8602777719497681, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.18507780134677887, | |
| "rewards/margins": 0.024849826470017433, | |
| "rewards/rejected": -0.20992763340473175, | |
| "sft_loss": 1.8507779836654663, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.15241320914479256, | |
| "grad_norm": 4.903147220611572, | |
| "learning_rate": 4.96823784759222e-06, | |
| "logits/chosen": 225.79443359375, | |
| "logits/rejected": 226.6028289794922, | |
| "logps/chosen": -1.7729737758636475, | |
| "logps/rejected": -2.0627281665802, | |
| "loss": 1.848, | |
| "odds_ratio_loss": 0.7500249147415161, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.17729738354682922, | |
| "rewards/margins": 0.02897545136511326, | |
| "rewards/rejected": -0.20627284049987793, | |
| "sft_loss": 1.7729737758636475, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.1693480101608806, | |
| "grad_norm": 3.5224053859710693, | |
| "learning_rate": 4.960798291629323e-06, | |
| "logits/chosen": 232.6717071533203, | |
| "logits/rejected": 232.73568725585938, | |
| "logps/chosen": -1.7854640483856201, | |
| "logps/rejected": -1.9538242816925049, | |
| "loss": 1.8613, | |
| "odds_ratio_loss": 0.7585908770561218, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.17854642868041992, | |
| "rewards/margins": 0.016835981979966164, | |
| "rewards/rejected": -0.19538240134716034, | |
| "sft_loss": 1.7854640483856201, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.18628281117696868, | |
| "grad_norm": 6.120446681976318, | |
| "learning_rate": 4.952583440813383e-06, | |
| "logits/chosen": 235.1363983154297, | |
| "logits/rejected": 236.200927734375, | |
| "logps/chosen": -1.8061208724975586, | |
| "logps/rejected": -2.0079829692840576, | |
| "loss": 1.8823, | |
| "odds_ratio_loss": 0.7622145414352417, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.18061205744743347, | |
| "rewards/margins": 0.020186223089694977, | |
| "rewards/rejected": -0.20079830288887024, | |
| "sft_loss": 1.8061208724975586, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.20321761219305673, | |
| "grad_norm": 1.8270832300186157, | |
| "learning_rate": 4.943595883301086e-06, | |
| "logits/chosen": 234.9897918701172, | |
| "logits/rejected": 235.3126983642578, | |
| "logps/chosen": -1.6784818172454834, | |
| "logps/rejected": -1.882794737815857, | |
| "loss": 1.7524, | |
| "odds_ratio_loss": 0.7392091155052185, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.16784818470478058, | |
| "rewards/margins": 0.02043129876255989, | |
| "rewards/rejected": -0.18827947974205017, | |
| "sft_loss": 1.6784818172454834, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.2201524132091448, | |
| "grad_norm": 5.280871868133545, | |
| "learning_rate": 4.933838450696757e-06, | |
| "logits/chosen": 238.5899200439453, | |
| "logits/rejected": 238.400390625, | |
| "logps/chosen": -1.512404203414917, | |
| "logps/rejected": -1.6778627634048462, | |
| "loss": 1.5856, | |
| "odds_ratio_loss": 0.7318023443222046, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.15124042332172394, | |
| "rewards/margins": 0.0165458582341671, | |
| "rewards/rejected": -0.16778628528118134, | |
| "sft_loss": 1.512404203414917, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.23708721422523285, | |
| "grad_norm": 4.924640655517578, | |
| "learning_rate": 4.923314217160234e-06, | |
| "logits/chosen": 240.5208282470703, | |
| "logits/rejected": 241.1940460205078, | |
| "logps/chosen": -1.6607558727264404, | |
| "logps/rejected": -1.8365033864974976, | |
| "loss": 1.7352, | |
| "odds_ratio_loss": 0.7449204921722412, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.16607561707496643, | |
| "rewards/margins": 0.017574718222022057, | |
| "rewards/rejected": -0.18365031480789185, | |
| "sft_loss": 1.6607558727264404, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.2540220152413209, | |
| "grad_norm": 1.817272663116455, | |
| "learning_rate": 4.9120264984383285e-06, | |
| "logits/chosen": 242.286376953125, | |
| "logits/rejected": 242.5799560546875, | |
| "logps/chosen": -1.3862061500549316, | |
| "logps/rejected": -1.5650876760482788, | |
| "loss": 1.454, | |
| "odds_ratio_loss": 0.6781331300735474, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.13862061500549316, | |
| "rewards/margins": 0.01788817159831524, | |
| "rewards/rejected": -0.15650877356529236, | |
| "sft_loss": 1.3862061500549316, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.270956816257409, | |
| "grad_norm": 1.3204820156097412, | |
| "learning_rate": 4.899978850820176e-06, | |
| "logits/chosen": 244.1859588623047, | |
| "logits/rejected": 244.36865234375, | |
| "logps/chosen": -1.5278394222259521, | |
| "logps/rejected": -1.7709777355194092, | |
| "loss": 1.5938, | |
| "odds_ratio_loss": 0.6600898504257202, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.15278393030166626, | |
| "rewards/margins": 0.02431386150419712, | |
| "rewards/rejected": -0.17709779739379883, | |
| "sft_loss": 1.5278394222259521, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.28789161727349705, | |
| "grad_norm": 2.3198039531707764, | |
| "learning_rate": 4.887175070016795e-06, | |
| "logits/chosen": 242.52523803710938, | |
| "logits/rejected": 243.25057983398438, | |
| "logps/chosen": -1.3887180089950562, | |
| "logps/rejected": -1.6768462657928467, | |
| "loss": 1.4533, | |
| "odds_ratio_loss": 0.6456118822097778, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.13887180387973785, | |
| "rewards/margins": 0.02881282940506935, | |
| "rewards/rejected": -0.1676846444606781, | |
| "sft_loss": 1.3887180089950562, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.3048264182895851, | |
| "grad_norm": 2.7539143562316895, | |
| "learning_rate": 4.873619189965217e-06, | |
| "logits/chosen": 246.211181640625, | |
| "logits/rejected": 246.160400390625, | |
| "logps/chosen": -1.5571125745773315, | |
| "logps/rejected": -1.8048861026763916, | |
| "loss": 1.6256, | |
| "odds_ratio_loss": 0.684849202632904, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.15571126341819763, | |
| "rewards/margins": 0.024777347221970558, | |
| "rewards/rejected": -0.18048861622810364, | |
| "sft_loss": 1.5571125745773315, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.32176121930567314, | |
| "grad_norm": 1.5920647382736206, | |
| "learning_rate": 4.859315481557563e-06, | |
| "logits/chosen": 243.8936309814453, | |
| "logits/rejected": 244.6537628173828, | |
| "logps/chosen": -1.3444526195526123, | |
| "logps/rejected": -1.622938871383667, | |
| "loss": 1.4127, | |
| "odds_ratio_loss": 0.6820069551467896, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.13444526493549347, | |
| "rewards/margins": 0.027848612517118454, | |
| "rewards/rejected": -0.16229388117790222, | |
| "sft_loss": 1.3444526195526123, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.3386960203217612, | |
| "grad_norm": 1.2194499969482422, | |
| "learning_rate": 4.84426845129546e-06, | |
| "logits/chosen": 245.6062774658203, | |
| "logits/rejected": 245.9265899658203, | |
| "logps/chosen": -1.4105961322784424, | |
| "logps/rejected": -1.591505765914917, | |
| "loss": 1.4778, | |
| "odds_ratio_loss": 0.6719616055488586, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.14105962216854095, | |
| "rewards/margins": 0.018090957775712013, | |
| "rewards/rejected": -0.15915057063102722, | |
| "sft_loss": 1.4105961322784424, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.3556308213378493, | |
| "grad_norm": 1.6798571348190308, | |
| "learning_rate": 4.828482839870233e-06, | |
| "logits/chosen": 244.5519561767578, | |
| "logits/rejected": 244.9039764404297, | |
| "logps/chosen": -1.4486945867538452, | |
| "logps/rejected": -1.5620288848876953, | |
| "loss": 1.5235, | |
| "odds_ratio_loss": 0.7479228973388672, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.14486947655677795, | |
| "rewards/margins": 0.011333415284752846, | |
| "rewards/rejected": -0.15620288252830505, | |
| "sft_loss": 1.4486945867538452, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.37256562235393736, | |
| "grad_norm": 1.2994054555892944, | |
| "learning_rate": 4.811963620669314e-06, | |
| "logits/chosen": 247.3550567626953, | |
| "logits/rejected": 247.1066436767578, | |
| "logps/chosen": -1.46903395652771, | |
| "logps/rejected": -1.5901343822479248, | |
| "loss": 1.5401, | |
| "odds_ratio_loss": 0.7104851007461548, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.14690342545509338, | |
| "rewards/margins": 0.012110031209886074, | |
| "rewards/rejected": -0.15901342034339905, | |
| "sft_loss": 1.46903395652771, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.3895004233700254, | |
| "grad_norm": 1.8322347402572632, | |
| "learning_rate": 4.794715998209328e-06, | |
| "logits/chosen": 245.6450653076172, | |
| "logits/rejected": 246.178955078125, | |
| "logps/chosen": -1.3713748455047607, | |
| "logps/rejected": -1.5887004137039185, | |
| "loss": 1.4399, | |
| "odds_ratio_loss": 0.6854357719421387, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.1371375024318695, | |
| "rewards/margins": 0.02173253521323204, | |
| "rewards/rejected": -0.15887005627155304, | |
| "sft_loss": 1.3713748455047607, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.40643522438611346, | |
| "grad_norm": 1.3373929262161255, | |
| "learning_rate": 4.7767454064963724e-06, | |
| "logits/chosen": 246.9872283935547, | |
| "logits/rejected": 247.23184204101562, | |
| "logps/chosen": -1.418143391609192, | |
| "logps/rejected": -1.6108840703964233, | |
| "loss": 1.485, | |
| "odds_ratio_loss": 0.6687358617782593, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.14181432127952576, | |
| "rewards/margins": 0.019274089485406876, | |
| "rewards/rejected": -0.16108840703964233, | |
| "sft_loss": 1.418143391609192, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.42337002540220153, | |
| "grad_norm": 2.7464914321899414, | |
| "learning_rate": 4.758057507313987e-06, | |
| "logits/chosen": 246.5187530517578, | |
| "logits/rejected": 247.2135772705078, | |
| "logps/chosen": -1.3453131914138794, | |
| "logps/rejected": -1.5121968984603882, | |
| "loss": 1.4146, | |
| "odds_ratio_loss": 0.6926182508468628, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.13453131914138794, | |
| "rewards/margins": 0.0166883897036314, | |
| "rewards/rejected": -0.1512196958065033, | |
| "sft_loss": 1.3453131914138794, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.4403048264182896, | |
| "grad_norm": 6.080187797546387, | |
| "learning_rate": 4.73865818843936e-06, | |
| "logits/chosen": 247.1885223388672, | |
| "logits/rejected": 247.33627319335938, | |
| "logps/chosen": -1.4415591955184937, | |
| "logps/rejected": -1.6857059001922607, | |
| "loss": 1.5104, | |
| "odds_ratio_loss": 0.6885947585105896, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.14415593445301056, | |
| "rewards/margins": 0.0244146715849638, | |
| "rewards/rejected": -0.1685706079006195, | |
| "sft_loss": 1.4415591955184937, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.4572396274343776, | |
| "grad_norm": 1.4364287853240967, | |
| "learning_rate": 4.718553561788339e-06, | |
| "logits/chosen": 247.25503540039062, | |
| "logits/rejected": 247.4564971923828, | |
| "logps/chosen": -1.3397562503814697, | |
| "logps/rejected": -1.5836654901504517, | |
| "loss": 1.4039, | |
| "odds_ratio_loss": 0.6409639120101929, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.13397563993930817, | |
| "rewards/margins": 0.024390894919633865, | |
| "rewards/rejected": -0.15836653113365173, | |
| "sft_loss": 1.3397562503814697, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.4741744284504657, | |
| "grad_norm": 1.979526162147522, | |
| "learning_rate": 4.697749961489822e-06, | |
| "logits/chosen": 246.6254425048828, | |
| "logits/rejected": 246.77536010742188, | |
| "logps/chosen": -1.5182511806488037, | |
| "logps/rejected": -1.7493031024932861, | |
| "loss": 1.5866, | |
| "odds_ratio_loss": 0.6837342381477356, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.15182514488697052, | |
| "rewards/margins": 0.023105164989829063, | |
| "rewards/rejected": -0.17493028938770294, | |
| "sft_loss": 1.5182511806488037, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.4911092294665538, | |
| "grad_norm": 2.0010595321655273, | |
| "learning_rate": 4.67625394189013e-06, | |
| "logits/chosen": 246.59414672851562, | |
| "logits/rejected": 246.92410278320312, | |
| "logps/chosen": -1.317578673362732, | |
| "logps/rejected": -1.5447911024093628, | |
| "loss": 1.381, | |
| "odds_ratio_loss": 0.6341525316238403, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.13175788521766663, | |
| "rewards/margins": 0.022721242159605026, | |
| "rewards/rejected": -0.15447911620140076, | |
| "sft_loss": 1.317578673362732, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.5080440304826418, | |
| "grad_norm": 3.1077401638031006, | |
| "learning_rate": 4.654072275488016e-06, | |
| "logits/chosen": 248.23709106445312, | |
| "logits/rejected": 248.05142211914062, | |
| "logps/chosen": -1.2916858196258545, | |
| "logps/rejected": -1.4983139038085938, | |
| "loss": 1.3555, | |
| "odds_ratio_loss": 0.6378514766693115, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.1291685849428177, | |
| "rewards/margins": 0.020662816241383553, | |
| "rewards/rejected": -0.14983141422271729, | |
| "sft_loss": 1.2916858196258545, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.5249788314987299, | |
| "grad_norm": 2.628451108932495, | |
| "learning_rate": 4.631211950800925e-06, | |
| "logits/chosen": 248.5096435546875, | |
| "logits/rejected": 248.2730712890625, | |
| "logps/chosen": -1.3065412044525146, | |
| "logps/rejected": -1.4278388023376465, | |
| "loss": 1.3779, | |
| "odds_ratio_loss": 0.7139269709587097, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.13065412640571594, | |
| "rewards/margins": 0.012129749171435833, | |
| "rewards/rejected": -0.14278386533260345, | |
| "sft_loss": 1.3065412044525146, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.541913632514818, | |
| "grad_norm": 2.022738218307495, | |
| "learning_rate": 4.6076801701632095e-06, | |
| "logits/chosen": 250.45730590820312, | |
| "logits/rejected": 249.79464721679688, | |
| "logps/chosen": -1.3189256191253662, | |
| "logps/rejected": -1.4538644552230835, | |
| "loss": 1.3894, | |
| "odds_ratio_loss": 0.7043124437332153, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.13189253211021423, | |
| "rewards/margins": 0.013493897393345833, | |
| "rewards/rejected": -0.1453864425420761, | |
| "sft_loss": 1.3189256191253662, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.558848433530906, | |
| "grad_norm": 1.9951550960540771, | |
| "learning_rate": 4.583484347456972e-06, | |
| "logits/chosen": 247.82229614257812, | |
| "logits/rejected": 248.18814086914062, | |
| "logps/chosen": -1.4347447156906128, | |
| "logps/rejected": -1.533195972442627, | |
| "loss": 1.5114, | |
| "odds_ratio_loss": 0.7667852640151978, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.14347447454929352, | |
| "rewards/margins": 0.00984511710703373, | |
| "rewards/rejected": -0.1533195823431015, | |
| "sft_loss": 1.4347447156906128, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.5757832345469941, | |
| "grad_norm": 1.1787029504776, | |
| "learning_rate": 4.55863210577626e-06, | |
| "logits/chosen": 248.49844360351562, | |
| "logits/rejected": 249.04006958007812, | |
| "logps/chosen": -1.3999149799346924, | |
| "logps/rejected": -1.6324024200439453, | |
| "loss": 1.4655, | |
| "odds_ratio_loss": 0.6553832292556763, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.13999152183532715, | |
| "rewards/margins": 0.02324872463941574, | |
| "rewards/rejected": -0.1632402390241623, | |
| "sft_loss": 1.3999149799346924, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.5927180355630821, | |
| "grad_norm": 1.3341506719589233, | |
| "learning_rate": 4.5331312750253465e-06, | |
| "logits/chosen": 250.0833740234375, | |
| "logits/rejected": 249.986328125, | |
| "logps/chosen": -1.353328824043274, | |
| "logps/rejected": -1.4628279209136963, | |
| "loss": 1.4282, | |
| "odds_ratio_loss": 0.7489296197891235, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.1353328824043274, | |
| "rewards/margins": 0.010949901305139065, | |
| "rewards/rejected": -0.14628279209136963, | |
| "sft_loss": 1.353328824043274, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.6096528365791702, | |
| "grad_norm": 7.017007827758789, | |
| "learning_rate": 4.506989889451858e-06, | |
| "logits/chosen": 250.78952026367188, | |
| "logits/rejected": 251.0829315185547, | |
| "logps/chosen": -1.3330605030059814, | |
| "logps/rejected": -1.492555022239685, | |
| "loss": 1.4015, | |
| "odds_ratio_loss": 0.6845325231552124, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.13330607116222382, | |
| "rewards/margins": 0.01594943180680275, | |
| "rewards/rejected": -0.14925549924373627, | |
| "sft_loss": 1.3330605030059814, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.6265876375952583, | |
| "grad_norm": 2.496628761291504, | |
| "learning_rate": 4.480216185115512e-06, | |
| "logits/chosen": 251.03042602539062, | |
| "logits/rejected": 251.23403930664062, | |
| "logps/chosen": -1.3291356563568115, | |
| "logps/rejected": -1.5497068166732788, | |
| "loss": 1.3933, | |
| "odds_ratio_loss": 0.641234278678894, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.13291357457637787, | |
| "rewards/margins": 0.022057104855775833, | |
| "rewards/rejected": -0.1549706757068634, | |
| "sft_loss": 1.3291356563568115, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.6435224386113463, | |
| "grad_norm": 1.7296651601791382, | |
| "learning_rate": 4.4528185972932856e-06, | |
| "logits/chosen": 249.67446899414062, | |
| "logits/rejected": 250.25466918945312, | |
| "logps/chosen": -1.3678677082061768, | |
| "logps/rejected": -1.639181137084961, | |
| "loss": 1.4346, | |
| "odds_ratio_loss": 0.6676440834999084, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.13678675889968872, | |
| "rewards/margins": 0.02713136002421379, | |
| "rewards/rejected": -0.1639181226491928, | |
| "sft_loss": 1.3678677082061768, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.6604572396274344, | |
| "grad_norm": 5.801638126373291, | |
| "learning_rate": 4.424805757821803e-06, | |
| "logits/chosen": 249.99923706054688, | |
| "logits/rejected": 250.46279907226562, | |
| "logps/chosen": -1.4075113534927368, | |
| "logps/rejected": -1.539193868637085, | |
| "loss": 1.4777, | |
| "odds_ratio_loss": 0.7016426920890808, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.14075112342834473, | |
| "rewards/margins": 0.013168272562325, | |
| "rewards/rejected": -0.15391941368579865, | |
| "sft_loss": 1.4075113534927368, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.6773920406435224, | |
| "grad_norm": 2.283698558807373, | |
| "learning_rate": 4.396186492377812e-06, | |
| "logits/chosen": 250.4367218017578, | |
| "logits/rejected": 250.3822479248047, | |
| "logps/chosen": -1.3555715084075928, | |
| "logps/rejected": -1.6237144470214844, | |
| "loss": 1.4187, | |
| "odds_ratio_loss": 0.631491482257843, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.1355571299791336, | |
| "rewards/margins": 0.026814306154847145, | |
| "rewards/rejected": -0.1623714417219162, | |
| "sft_loss": 1.3555715084075928, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.6943268416596104, | |
| "grad_norm": 3.2836830615997314, | |
| "learning_rate": 4.366969817697578e-06, | |
| "logits/chosen": 249.8821563720703, | |
| "logits/rejected": 250.1019744873047, | |
| "logps/chosen": -1.3262561559677124, | |
| "logps/rejected": -1.481173038482666, | |
| "loss": 1.397, | |
| "odds_ratio_loss": 0.7070230841636658, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.13262560963630676, | |
| "rewards/margins": 0.015491697005927563, | |
| "rewards/rejected": -0.1481173038482666, | |
| "sft_loss": 1.3262561559677124, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.7112616426756986, | |
| "grad_norm": 2.213815689086914, | |
| "learning_rate": 4.337164938736086e-06, | |
| "logits/chosen": 250.5995635986328, | |
| "logits/rejected": 250.66165161132812, | |
| "logps/chosen": -1.3863251209259033, | |
| "logps/rejected": -1.513422966003418, | |
| "loss": 1.4589, | |
| "odds_ratio_loss": 0.7256019115447998, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.13863249123096466, | |
| "rewards/margins": 0.012709791772067547, | |
| "rewards/rejected": -0.15134228765964508, | |
| "sft_loss": 1.3863251209259033, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.7281964436917866, | |
| "grad_norm": 1.0603761672973633, | |
| "learning_rate": 4.306781245766945e-06, | |
| "logits/chosen": 250.349853515625, | |
| "logits/rejected": 250.0968780517578, | |
| "logps/chosen": -1.2664977312088013, | |
| "logps/rejected": -1.475754976272583, | |
| "loss": 1.3367, | |
| "odds_ratio_loss": 0.7022702097892761, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12664976716041565, | |
| "rewards/margins": 0.0209257360547781, | |
| "rewards/rejected": -0.1475754976272583, | |
| "sft_loss": 1.2664977312088013, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.7451312447078747, | |
| "grad_norm": 4.412312984466553, | |
| "learning_rate": 4.275828311423903e-06, | |
| "logits/chosen": 249.72463989257812, | |
| "logits/rejected": 250.4665985107422, | |
| "logps/chosen": -1.4287374019622803, | |
| "logps/rejected": -1.5396887063980103, | |
| "loss": 1.4987, | |
| "odds_ratio_loss": 0.6998986005783081, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.14287373423576355, | |
| "rewards/margins": 0.011095147579908371, | |
| "rewards/rejected": -0.15396887063980103, | |
| "sft_loss": 1.4287374019622803, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.7620660457239627, | |
| "grad_norm": 6.307819843292236, | |
| "learning_rate": 4.244315887684912e-06, | |
| "logits/chosen": 250.8037109375, | |
| "logits/rejected": 251.0100555419922, | |
| "logps/chosen": -1.242921233177185, | |
| "logps/rejected": -1.4474495649337769, | |
| "loss": 1.3115, | |
| "odds_ratio_loss": 0.6855745911598206, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.12429212033748627, | |
| "rewards/margins": 0.020452830940485, | |
| "rewards/rejected": -0.14474496245384216, | |
| "sft_loss": 1.242921233177185, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.7790008467400508, | |
| "grad_norm": 1.9661298990249634, | |
| "learning_rate": 4.212253902799685e-06, | |
| "logits/chosen": 250.1886749267578, | |
| "logits/rejected": 250.97207641601562, | |
| "logps/chosen": -1.3525545597076416, | |
| "logps/rejected": -1.6002397537231445, | |
| "loss": 1.4203, | |
| "odds_ratio_loss": 0.6769625544548035, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.13525545597076416, | |
| "rewards/margins": 0.024768516421318054, | |
| "rewards/rejected": -0.16002397239208221, | |
| "sft_loss": 1.3525545597076416, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.7959356477561389, | |
| "grad_norm": 2.6013875007629395, | |
| "learning_rate": 4.179652458161718e-06, | |
| "logits/chosen": 250.9954376220703, | |
| "logits/rejected": 250.8687286376953, | |
| "logps/chosen": -1.3839843273162842, | |
| "logps/rejected": -1.4946000576019287, | |
| "loss": 1.4574, | |
| "odds_ratio_loss": 0.7343354821205139, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.1383984386920929, | |
| "rewards/margins": 0.011061567813158035, | |
| "rewards/rejected": -0.14946000277996063, | |
| "sft_loss": 1.3839843273162842, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.8128704487722269, | |
| "grad_norm": 1.4564313888549805, | |
| "learning_rate": 4.146521825125765e-06, | |
| "logits/chosen": 251.6759796142578, | |
| "logits/rejected": 251.0865478515625, | |
| "logps/chosen": -1.3470927476882935, | |
| "logps/rejected": -1.5811121463775635, | |
| "loss": 1.4152, | |
| "odds_ratio_loss": 0.6814435720443726, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.13470926880836487, | |
| "rewards/margins": 0.023401936516165733, | |
| "rewards/rejected": -0.15811121463775635, | |
| "sft_loss": 1.3470927476882935, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.8298052497883149, | |
| "grad_norm": 2.20600962638855, | |
| "learning_rate": 4.11287244177176e-06, | |
| "logits/chosen": 252.32333374023438, | |
| "logits/rejected": 252.3999481201172, | |
| "logps/chosen": -1.313819408416748, | |
| "logps/rejected": -1.5711250305175781, | |
| "loss": 1.3752, | |
| "odds_ratio_loss": 0.6139523386955261, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.13138195872306824, | |
| "rewards/margins": 0.0257305596023798, | |
| "rewards/rejected": -0.1571125090122223, | |
| "sft_loss": 1.313819408416748, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.8467400508044031, | |
| "grad_norm": 6.151379108428955, | |
| "learning_rate": 4.078714909616215e-06, | |
| "logits/chosen": 250.7323760986328, | |
| "logits/rejected": 250.898681640625, | |
| "logps/chosen": -1.3562355041503906, | |
| "logps/rejected": -1.6217498779296875, | |
| "loss": 1.422, | |
| "odds_ratio_loss": 0.6572010517120361, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.13562354445457458, | |
| "rewards/margins": 0.026551440358161926, | |
| "rewards/rejected": -0.1621749848127365, | |
| "sft_loss": 1.3562355041503906, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8467400508044031, | |
| "eval_logits/chosen": 250.27386474609375, | |
| "eval_logits/rejected": 250.56336975097656, | |
| "eval_logps/chosen": -1.3222252130508423, | |
| "eval_logps/rejected": -1.5458606481552124, | |
| "eval_loss": 1.3895596265792847, | |
| "eval_odds_ratio_loss": 0.6733438968658447, | |
| "eval_rewards/accuracies": 0.5752381086349487, | |
| "eval_rewards/chosen": -0.1322225034236908, | |
| "eval_rewards/margins": 0.022363554686307907, | |
| "eval_rewards/rejected": -0.1545860767364502, | |
| "eval_runtime": 222.405, | |
| "eval_samples_per_second": 4.721, | |
| "eval_sft_loss": 1.3222252130508423, | |
| "eval_steps_per_second": 2.361, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8636748518204911, | |
| "grad_norm": 7.651697635650635, | |
| "learning_rate": 4.044059990272125e-06, | |
| "logits/chosen": 250.82901000976562, | |
| "logits/rejected": 250.9202423095703, | |
| "logps/chosen": -1.3385546207427979, | |
| "logps/rejected": -1.5796321630477905, | |
| "loss": 1.4082, | |
| "odds_ratio_loss": 0.6965717077255249, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.13385547697544098, | |
| "rewards/margins": 0.024107756093144417, | |
| "rewards/rejected": -0.15796321630477905, | |
| "sft_loss": 1.3385546207427979, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.8806096528365792, | |
| "grad_norm": 2.0727665424346924, | |
| "learning_rate": 4.0089186020584345e-06, | |
| "logits/chosen": 250.8845977783203, | |
| "logits/rejected": 251.09323120117188, | |
| "logps/chosen": -1.3856604099273682, | |
| "logps/rejected": -1.521828055381775, | |
| "loss": 1.453, | |
| "odds_ratio_loss": 0.6737630367279053, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.1385660469532013, | |
| "rewards/margins": 0.013616764917969704, | |
| "rewards/rejected": -0.15218280255794525, | |
| "sft_loss": 1.3856604099273682, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.8975444538526672, | |
| "grad_norm": 1.6497074365615845, | |
| "learning_rate": 3.973301816560124e-06, | |
| "logits/chosen": 250.5475616455078, | |
| "logits/rejected": 251.07559204101562, | |
| "logps/chosen": -1.2916876077651978, | |
| "logps/rejected": -1.5535304546356201, | |
| "loss": 1.355, | |
| "odds_ratio_loss": 0.6329156160354614, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.12916876375675201, | |
| "rewards/margins": 0.026184294372797012, | |
| "rewards/rejected": -0.15535303950309753, | |
| "sft_loss": 1.2916876077651978, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.9144792548687553, | |
| "grad_norm": 2.159715414047241, | |
| "learning_rate": 3.937220855140021e-06, | |
| "logits/chosen": 252.92514038085938, | |
| "logits/rejected": 252.7586212158203, | |
| "logps/chosen": -1.2703886032104492, | |
| "logps/rejected": -1.4452732801437378, | |
| "loss": 1.3383, | |
| "odds_ratio_loss": 0.6788212060928345, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.1270388662815094, | |
| "rewards/margins": 0.017488475888967514, | |
| "rewards/rejected": -0.14452733099460602, | |
| "sft_loss": 1.2703886032104492, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.9314140558848434, | |
| "grad_norm": 1.6743594408035278, | |
| "learning_rate": 3.900687085403418e-06, | |
| "logits/chosen": 251.9027099609375, | |
| "logits/rejected": 251.82095336914062, | |
| "logps/chosen": -1.2356189489364624, | |
| "logps/rejected": -1.3227603435516357, | |
| "loss": 1.3057, | |
| "odds_ratio_loss": 0.7003273963928223, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.12356189638376236, | |
| "rewards/margins": 0.008714134804904461, | |
| "rewards/rejected": -0.1322760283946991, | |
| "sft_loss": 1.2356189489364624, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.9483488569009314, | |
| "grad_norm": 2.0650203227996826, | |
| "learning_rate": 3.863712017616614e-06, | |
| "logits/chosen": 252.4972686767578, | |
| "logits/rejected": 253.4659881591797, | |
| "logps/chosen": -1.33904230594635, | |
| "logps/rejected": -1.5850245952606201, | |
| "loss": 1.4043, | |
| "odds_ratio_loss": 0.6524351835250854, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.13390421867370605, | |
| "rewards/margins": 0.024598244577646255, | |
| "rewards/rejected": -0.158502459526062, | |
| "sft_loss": 1.33904230594635, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.9652836579170194, | |
| "grad_norm": 4.395691394805908, | |
| "learning_rate": 3.826307301080504e-06, | |
| "logits/chosen": 250.24227905273438, | |
| "logits/rejected": 250.7295379638672, | |
| "logps/chosen": -1.3605867624282837, | |
| "logps/rejected": -1.592639684677124, | |
| "loss": 1.436, | |
| "odds_ratio_loss": 0.7545104622840881, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.13605865836143494, | |
| "rewards/margins": 0.02320530079305172, | |
| "rewards/rejected": -0.1592639982700348, | |
| "sft_loss": 1.3605867624282837, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.9822184589331076, | |
| "grad_norm": 2.8619537353515625, | |
| "learning_rate": 3.7884847204603775e-06, | |
| "logits/chosen": 252.0938262939453, | |
| "logits/rejected": 252.2036895751953, | |
| "logps/chosen": -1.2960541248321533, | |
| "logps/rejected": -1.4377543926239014, | |
| "loss": 1.3661, | |
| "odds_ratio_loss": 0.7007730007171631, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.12960541248321533, | |
| "rewards/margins": 0.014170033857226372, | |
| "rewards/rejected": -0.14377544820308685, | |
| "sft_loss": 1.2960541248321533, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.9991532599491956, | |
| "grad_norm": 2.728938102722168, | |
| "learning_rate": 3.750256192073058e-06, | |
| "logits/chosen": 252.297607421875, | |
| "logits/rejected": 252.3206024169922, | |
| "logps/chosen": -1.4331130981445312, | |
| "logps/rejected": -1.4828779697418213, | |
| "loss": 1.5094, | |
| "odds_ratio_loss": 0.7630207538604736, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.1433113068342209, | |
| "rewards/margins": 0.004976483527570963, | |
| "rewards/rejected": -0.1482878029346466, | |
| "sft_loss": 1.4331130981445312, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 1.0160880609652836, | |
| "grad_norm": 4.769712924957275, | |
| "learning_rate": 3.7116337601325715e-06, | |
| "logits/chosen": 251.11611938476562, | |
| "logits/rejected": 251.3095245361328, | |
| "logps/chosen": -1.2182286977767944, | |
| "logps/rejected": -1.3984460830688477, | |
| "loss": 1.285, | |
| "odds_ratio_loss": 0.6677871346473694, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12182287871837616, | |
| "rewards/margins": 0.01802174374461174, | |
| "rewards/rejected": -0.139844611287117, | |
| "sft_loss": 1.2182286977767944, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 1.0330228619813717, | |
| "grad_norm": 2.1474721431732178, | |
| "learning_rate": 3.6726295929555154e-06, | |
| "logits/chosen": 251.42453002929688, | |
| "logits/rejected": 252.20944213867188, | |
| "logps/chosen": -1.2515238523483276, | |
| "logps/rejected": -1.3369293212890625, | |
| "loss": 1.3216, | |
| "odds_ratio_loss": 0.7008894085884094, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12515239417552948, | |
| "rewards/margins": 0.008540543727576733, | |
| "rewards/rejected": -0.1336929351091385, | |
| "sft_loss": 1.2515238523483276, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.0499576629974599, | |
| "grad_norm": 2.5692148208618164, | |
| "learning_rate": 3.6332559791273307e-06, | |
| "logits/chosen": 251.50045776367188, | |
| "logits/rejected": 251.7969970703125, | |
| "logps/chosen": -1.236567735671997, | |
| "logps/rejected": -1.4078432321548462, | |
| "loss": 1.3014, | |
| "odds_ratio_loss": 0.6482684016227722, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.12365677207708359, | |
| "rewards/margins": 0.01712755486369133, | |
| "rewards/rejected": -0.14078432321548462, | |
| "sft_loss": 1.236567735671997, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 1.0668924640135478, | |
| "grad_norm": 3.342146635055542, | |
| "learning_rate": 3.593525323630681e-06, | |
| "logits/chosen": 251.15097045898438, | |
| "logits/rejected": 251.6619110107422, | |
| "logps/chosen": -1.2767521142959595, | |
| "logps/rejected": -1.4714807271957397, | |
| "loss": 1.34, | |
| "odds_ratio_loss": 0.6320163607597351, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12767520546913147, | |
| "rewards/margins": 0.019472863525152206, | |
| "rewards/rejected": -0.14714807271957397, | |
| "sft_loss": 1.2767521142959595, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 1.083827265029636, | |
| "grad_norm": 1.7750215530395508, | |
| "learning_rate": 3.5534501439371615e-06, | |
| "logits/chosen": 253.18130493164062, | |
| "logits/rejected": 253.9321746826172, | |
| "logps/chosen": -1.2172341346740723, | |
| "logps/rejected": -1.4529893398284912, | |
| "loss": 1.282, | |
| "odds_ratio_loss": 0.6480832099914551, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.12172339856624603, | |
| "rewards/margins": 0.023575523868203163, | |
| "rewards/rejected": -0.14529892802238464, | |
| "sft_loss": 1.2172341346740723, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 1.100762066045724, | |
| "grad_norm": 3.0737292766571045, | |
| "learning_rate": 3.5130430660635633e-06, | |
| "logits/chosen": 253.6792449951172, | |
| "logits/rejected": 253.87606811523438, | |
| "logps/chosen": -1.2902581691741943, | |
| "logps/rejected": -1.4731221199035645, | |
| "loss": 1.3556, | |
| "odds_ratio_loss": 0.6539097428321838, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12902581691741943, | |
| "rewards/margins": 0.01828640140593052, | |
| "rewards/rejected": -0.1473122239112854, | |
| "sft_loss": 1.2902581691741943, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 1.117696867061812, | |
| "grad_norm": 1.3352795839309692, | |
| "learning_rate": 3.4723168205939444e-06, | |
| "logits/chosen": 252.404541015625, | |
| "logits/rejected": 252.8050994873047, | |
| "logps/chosen": -1.2593786716461182, | |
| "logps/rejected": -1.3454132080078125, | |
| "loss": 1.3296, | |
| "odds_ratio_loss": 0.7021333575248718, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12593786418437958, | |
| "rewards/margins": 0.008603455498814583, | |
| "rewards/rejected": -0.134541317820549, | |
| "sft_loss": 1.2593786716461182, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 1.1346316680779, | |
| "grad_norm": 1.4886754751205444, | |
| "learning_rate": 3.431284238668754e-06, | |
| "logits/chosen": 252.7022247314453, | |
| "logits/rejected": 253.0660858154297, | |
| "logps/chosen": -1.344042420387268, | |
| "logps/rejected": -1.5089060068130493, | |
| "loss": 1.4141, | |
| "odds_ratio_loss": 0.7006558775901794, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.134404256939888, | |
| "rewards/margins": 0.01648634299635887, | |
| "rewards/rejected": -0.15089061856269836, | |
| "sft_loss": 1.344042420387268, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 1.1515664690939882, | |
| "grad_norm": 8.335692405700684, | |
| "learning_rate": 3.389958247942274e-06, | |
| "logits/chosen": 251.40444946289062, | |
| "logits/rejected": 252.54983520507812, | |
| "logps/chosen": -1.358946442604065, | |
| "logps/rejected": -1.5931769609451294, | |
| "loss": 1.4306, | |
| "odds_ratio_loss": 0.7168671488761902, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.13589465618133545, | |
| "rewards/margins": 0.023423049598932266, | |
| "rewards/rejected": -0.15931770205497742, | |
| "sft_loss": 1.358946442604065, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 1.168501270110076, | |
| "grad_norm": 2.2777557373046875, | |
| "learning_rate": 3.3483518685096588e-06, | |
| "logits/chosen": 252.7519989013672, | |
| "logits/rejected": 252.7262420654297, | |
| "logps/chosen": -1.313045859336853, | |
| "logps/rejected": -1.4913846254348755, | |
| "loss": 1.3792, | |
| "odds_ratio_loss": 0.6616145372390747, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.13130459189414978, | |
| "rewards/margins": 0.017833886668086052, | |
| "rewards/rejected": -0.14913848042488098, | |
| "sft_loss": 1.313045859336853, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 1.1854360711261642, | |
| "grad_norm": 1.7483341693878174, | |
| "learning_rate": 3.306478208804839e-06, | |
| "logits/chosen": 251.73300170898438, | |
| "logits/rejected": 252.00765991210938, | |
| "logps/chosen": -1.3198022842407227, | |
| "logps/rejected": -1.5368399620056152, | |
| "loss": 1.3902, | |
| "odds_ratio_loss": 0.7039870023727417, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.13198022544384003, | |
| "rewards/margins": 0.02170378342270851, | |
| "rewards/rejected": -0.15368400514125824, | |
| "sft_loss": 1.3198022842407227, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.2023708721422524, | |
| "grad_norm": 3.1045162677764893, | |
| "learning_rate": 3.264350461470608e-06, | |
| "logits/chosen": 251.3062744140625, | |
| "logits/rejected": 251.24673461914062, | |
| "logps/chosen": -1.191873550415039, | |
| "logps/rejected": -1.5103055238723755, | |
| "loss": 1.2531, | |
| "odds_ratio_loss": 0.6123217344284058, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.1191873699426651, | |
| "rewards/margins": 0.031843192875385284, | |
| "rewards/rejected": -0.15103057026863098, | |
| "sft_loss": 1.191873550415039, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 1.2193056731583405, | |
| "grad_norm": 4.187674522399902, | |
| "learning_rate": 3.2219818992021685e-06, | |
| "logits/chosen": 250.6509246826172, | |
| "logits/rejected": 250.8859100341797, | |
| "logps/chosen": -1.1646994352340698, | |
| "logps/rejected": -1.4936308860778809, | |
| "loss": 1.2266, | |
| "odds_ratio_loss": 0.6186606884002686, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.11646995693445206, | |
| "rewards/margins": 0.032893143594264984, | |
| "rewards/rejected": -0.14936310052871704, | |
| "sft_loss": 1.1646994352340698, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 1.2362404741744284, | |
| "grad_norm": 6.236975193023682, | |
| "learning_rate": 3.1793858705654595e-06, | |
| "logits/chosen": 252.42489624023438, | |
| "logits/rejected": 252.6690673828125, | |
| "logps/chosen": -1.1537976264953613, | |
| "logps/rejected": -1.407354712486267, | |
| "loss": 1.2166, | |
| "odds_ratio_loss": 0.6279497146606445, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.11537975072860718, | |
| "rewards/margins": 0.02535572089254856, | |
| "rewards/rejected": -0.1407354772090912, | |
| "sft_loss": 1.1537976264953613, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 1.2531752751905165, | |
| "grad_norm": 4.287051677703857, | |
| "learning_rate": 3.1365757957915787e-06, | |
| "logits/chosen": 252.34127807617188, | |
| "logits/rejected": 252.6292724609375, | |
| "logps/chosen": -1.2895433902740479, | |
| "logps/rejected": -1.4524219036102295, | |
| "loss": 1.3549, | |
| "odds_ratio_loss": 0.6531358957290649, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.12895433604717255, | |
| "rewards/margins": 0.016287846490740776, | |
| "rewards/rejected": -0.14524218440055847, | |
| "sft_loss": 1.2895433902740479, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 1.2701100762066047, | |
| "grad_norm": 3.4889471530914307, | |
| "learning_rate": 3.093565162548633e-06, | |
| "logits/chosen": 252.25082397460938, | |
| "logits/rejected": 252.3234405517578, | |
| "logps/chosen": -1.3181928396224976, | |
| "logps/rejected": -1.5464909076690674, | |
| "loss": 1.3886, | |
| "odds_ratio_loss": 0.7041035890579224, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.13181930780410767, | |
| "rewards/margins": 0.02282978780567646, | |
| "rewards/rejected": -0.15464909374713898, | |
| "sft_loss": 1.3181928396224976, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 1.2870448772226926, | |
| "grad_norm": 2.6276607513427734, | |
| "learning_rate": 3.0503675216923294e-06, | |
| "logits/chosen": 252.7392120361328, | |
| "logits/rejected": 253.15933227539062, | |
| "logps/chosen": -1.1722157001495361, | |
| "logps/rejected": -1.3820818662643433, | |
| "loss": 1.2364, | |
| "odds_ratio_loss": 0.642305850982666, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11722154915332794, | |
| "rewards/margins": 0.020986629649996758, | |
| "rewards/rejected": -0.13820818066596985, | |
| "sft_loss": 1.1722157001495361, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 1.3039796782387807, | |
| "grad_norm": 2.102112054824829, | |
| "learning_rate": 3.0069964829966748e-06, | |
| "logits/chosen": 251.8651885986328, | |
| "logits/rejected": 252.33065795898438, | |
| "logps/chosen": -1.238761305809021, | |
| "logps/rejected": -1.385432481765747, | |
| "loss": 1.3081, | |
| "odds_ratio_loss": 0.6930680274963379, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.12387613952159882, | |
| "rewards/margins": 0.014667103998363018, | |
| "rewards/rejected": -0.1385432332754135, | |
| "sft_loss": 1.238761305809021, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 1.3209144792548688, | |
| "grad_norm": 1.8301111459732056, | |
| "learning_rate": 2.963465710866094e-06, | |
| "logits/chosen": 251.42507934570312, | |
| "logits/rejected": 251.70413208007812, | |
| "logps/chosen": -1.2377374172210693, | |
| "logps/rejected": -1.572333574295044, | |
| "loss": 1.2996, | |
| "odds_ratio_loss": 0.6182882189750671, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.1237737163901329, | |
| "rewards/margins": 0.0334596149623394, | |
| "rewards/rejected": -0.1572333723306656, | |
| "sft_loss": 1.2377374172210693, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 1.337849280270957, | |
| "grad_norm": 7.058846473693848, | |
| "learning_rate": 2.919788920030357e-06, | |
| "logits/chosen": 253.6296844482422, | |
| "logits/rejected": 254.1488800048828, | |
| "logps/chosen": -1.2885361909866333, | |
| "logps/rejected": -1.4078810214996338, | |
| "loss": 1.3598, | |
| "odds_ratio_loss": 0.7127668857574463, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.12885363399982452, | |
| "rewards/margins": 0.011934494599699974, | |
| "rewards/rejected": -0.14078812301158905, | |
| "sft_loss": 1.2885361909866333, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 1.3547840812870449, | |
| "grad_norm": 3.2417685985565186, | |
| "learning_rate": 2.8759798712236303e-06, | |
| "logits/chosen": 253.9468231201172, | |
| "logits/rejected": 254.49008178710938, | |
| "logps/chosen": -1.2510448694229126, | |
| "logps/rejected": -1.554084300994873, | |
| "loss": 1.3164, | |
| "odds_ratio_loss": 0.6536312103271484, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12510448694229126, | |
| "rewards/margins": 0.030303955078125, | |
| "rewards/rejected": -0.15540844202041626, | |
| "sft_loss": 1.2510448694229126, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 1.371718882303133, | |
| "grad_norm": 1.9042048454284668, | |
| "learning_rate": 2.8320523668490507e-06, | |
| "logits/chosen": 253.7815704345703, | |
| "logits/rejected": 254.0332489013672, | |
| "logps/chosen": -1.2883363962173462, | |
| "logps/rejected": -1.4249004125595093, | |
| "loss": 1.3613, | |
| "odds_ratio_loss": 0.7295271754264832, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.12883363664150238, | |
| "rewards/margins": 0.01365639828145504, | |
| "rewards/rejected": -0.14249004423618317, | |
| "sft_loss": 1.2883363962173462, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 1.388653683319221, | |
| "grad_norm": 3.2637712955474854, | |
| "learning_rate": 2.7880202466301597e-06, | |
| "logits/chosen": 253.0078125, | |
| "logits/rejected": 253.09902954101562, | |
| "logps/chosen": -1.2434396743774414, | |
| "logps/rejected": -1.434788465499878, | |
| "loss": 1.3141, | |
| "odds_ratio_loss": 0.7062419652938843, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12434396892786026, | |
| "rewards/margins": 0.019134875386953354, | |
| "rewards/rejected": -0.14347884058952332, | |
| "sft_loss": 1.2434396743774414, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 1.405588484335309, | |
| "grad_norm": 2.8521387577056885, | |
| "learning_rate": 2.7438973832505854e-06, | |
| "logits/chosen": 251.6912841796875, | |
| "logits/rejected": 252.3306121826172, | |
| "logps/chosen": -1.175612211227417, | |
| "logps/rejected": -1.4439500570297241, | |
| "loss": 1.2387, | |
| "odds_ratio_loss": 0.6307461857795715, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.1175612211227417, | |
| "rewards/margins": 0.026833802461624146, | |
| "rewards/rejected": -0.14439500868320465, | |
| "sft_loss": 1.175612211227417, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 1.4225232853513972, | |
| "grad_norm": 8.145666122436523, | |
| "learning_rate": 2.699697677983341e-06, | |
| "logits/chosen": 253.7397918701172, | |
| "logits/rejected": 253.99203491210938, | |
| "logps/chosen": -1.2202482223510742, | |
| "logps/rejected": -1.2863436937332153, | |
| "loss": 1.2952, | |
| "odds_ratio_loss": 0.7496277093887329, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.12202481925487518, | |
| "rewards/margins": 0.006609548814594746, | |
| "rewards/rejected": -0.12863437831401825, | |
| "sft_loss": 1.2202482223510742, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 1.4394580863674853, | |
| "grad_norm": 8.698111534118652, | |
| "learning_rate": 2.6554350563111115e-06, | |
| "logits/chosen": 252.4715118408203, | |
| "logits/rejected": 252.827392578125, | |
| "logps/chosen": -1.286290168762207, | |
| "logps/rejected": -1.4043002128601074, | |
| "loss": 1.3605, | |
| "odds_ratio_loss": 0.7424478530883789, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.12862902879714966, | |
| "rewards/margins": 0.011800997890532017, | |
| "rewards/rejected": -0.1404300183057785, | |
| "sft_loss": 1.286290168762207, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 1.4563928873835732, | |
| "grad_norm": 5.0890398025512695, | |
| "learning_rate": 2.611123463538913e-06, | |
| "logits/chosen": 251.801513671875, | |
| "logits/rejected": 251.6995391845703, | |
| "logps/chosen": -1.1361697912216187, | |
| "logps/rejected": -1.4048140048980713, | |
| "loss": 1.1998, | |
| "odds_ratio_loss": 0.6365838050842285, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.11361698061227798, | |
| "rewards/margins": 0.02686440572142601, | |
| "rewards/rejected": -0.1404813826084137, | |
| "sft_loss": 1.1361697912216187, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 1.4733276883996613, | |
| "grad_norm": 2.2515621185302734, | |
| "learning_rate": 2.566776860400514e-06, | |
| "logits/chosen": 253.1812286376953, | |
| "logits/rejected": 253.3629913330078, | |
| "logps/chosen": -1.3416874408721924, | |
| "logps/rejected": -1.5248029232025146, | |
| "loss": 1.4085, | |
| "odds_ratio_loss": 0.6685901880264282, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.13416874408721924, | |
| "rewards/margins": 0.01831156201660633, | |
| "rewards/rejected": -0.15248030424118042, | |
| "sft_loss": 1.3416874408721924, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 1.4902624894157492, | |
| "grad_norm": 1.626927137374878, | |
| "learning_rate": 2.522409218659989e-06, | |
| "logits/chosen": 254.90966796875, | |
| "logits/rejected": 255.2332305908203, | |
| "logps/chosen": -1.2846667766571045, | |
| "logps/rejected": -1.4117066860198975, | |
| "loss": 1.3537, | |
| "odds_ratio_loss": 0.6902014017105103, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.1284666806459427, | |
| "rewards/margins": 0.012703998014330864, | |
| "rewards/rejected": -0.1411706656217575, | |
| "sft_loss": 1.2846667766571045, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 1.5071972904318374, | |
| "grad_norm": 2.0912625789642334, | |
| "learning_rate": 2.4780345167097976e-06, | |
| "logits/chosen": 252.5232696533203, | |
| "logits/rejected": 253.27197265625, | |
| "logps/chosen": -1.2608062028884888, | |
| "logps/rejected": -1.5506179332733154, | |
| "loss": 1.3254, | |
| "odds_ratio_loss": 0.6455532908439636, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12608060240745544, | |
| "rewards/margins": 0.02898118831217289, | |
| "rewards/rejected": -0.15506179630756378, | |
| "sft_loss": 1.2608062028884888, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 1.5241320914479255, | |
| "grad_norm": 2.230842351913452, | |
| "learning_rate": 2.4336667351667747e-06, | |
| "logits/chosen": 254.10025024414062, | |
| "logits/rejected": 254.30361938476562, | |
| "logps/chosen": -1.2958338260650635, | |
| "logps/rejected": -1.5468194484710693, | |
| "loss": 1.358, | |
| "odds_ratio_loss": 0.6219068765640259, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.12958340346813202, | |
| "rewards/margins": 0.025098532438278198, | |
| "rewards/rejected": -0.1546819508075714, | |
| "sft_loss": 1.2958338260650635, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 1.5410668924640136, | |
| "grad_norm": 4.812232494354248, | |
| "learning_rate": 2.3893198524674264e-06, | |
| "logits/chosen": 252.809814453125, | |
| "logits/rejected": 252.73892211914062, | |
| "logps/chosen": -1.2310715913772583, | |
| "logps/rejected": -1.428946852684021, | |
| "loss": 1.2975, | |
| "odds_ratio_loss": 0.6640914678573608, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12310715764760971, | |
| "rewards/margins": 0.019787531346082687, | |
| "rewards/rejected": -0.1428947001695633, | |
| "sft_loss": 1.2310715913772583, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 1.5580016934801018, | |
| "grad_norm": 2.730430841445923, | |
| "learning_rate": 2.345007840463904e-06, | |
| "logits/chosen": 254.012939453125, | |
| "logits/rejected": 254.33950805664062, | |
| "logps/chosen": -1.2894508838653564, | |
| "logps/rejected": -1.4280718564987183, | |
| "loss": 1.3585, | |
| "odds_ratio_loss": 0.6902931332588196, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.12894509732723236, | |
| "rewards/margins": 0.013862092979252338, | |
| "rewards/rejected": -0.14280718564987183, | |
| "sft_loss": 1.2894508838653564, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 1.5749364944961897, | |
| "grad_norm": 3.1161863803863525, | |
| "learning_rate": 2.3007446600220572e-06, | |
| "logits/chosen": 254.1707305908203, | |
| "logits/rejected": 253.95748901367188, | |
| "logps/chosen": -1.2720592021942139, | |
| "logps/rejected": -1.4615156650543213, | |
| "loss": 1.3436, | |
| "odds_ratio_loss": 0.7155615091323853, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.12720590829849243, | |
| "rewards/margins": 0.018945645540952682, | |
| "rewards/rejected": -0.1461515724658966, | |
| "sft_loss": 1.2720592021942139, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 1.5918712955122776, | |
| "grad_norm": 2.035860538482666, | |
| "learning_rate": 2.2565442566229507e-06, | |
| "logits/chosen": 253.0424346923828, | |
| "logits/rejected": 253.29043579101562, | |
| "logps/chosen": -1.2878587245941162, | |
| "logps/rejected": -1.4205825328826904, | |
| "loss": 1.3592, | |
| "odds_ratio_loss": 0.7135877013206482, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.1287858784198761, | |
| "rewards/margins": 0.013272365555167198, | |
| "rewards/rejected": -0.14205823838710785, | |
| "sft_loss": 1.2878587245941162, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 1.6088060965283657, | |
| "grad_norm": 2.338395357131958, | |
| "learning_rate": 2.2124205559692195e-06, | |
| "logits/chosen": 253.37295532226562, | |
| "logits/rejected": 253.95040893554688, | |
| "logps/chosen": -1.2591735124588013, | |
| "logps/rejected": -1.4865353107452393, | |
| "loss": 1.3211, | |
| "odds_ratio_loss": 0.6195861101150513, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.12591736018657684, | |
| "rewards/margins": 0.02273617871105671, | |
| "rewards/rejected": -0.1486535370349884, | |
| "sft_loss": 1.2591735124588013, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 1.6257408975444538, | |
| "grad_norm": 3.2883460521698, | |
| "learning_rate": 2.168387459597666e-06, | |
| "logits/chosen": 253.46572875976562, | |
| "logits/rejected": 253.340576171875, | |
| "logps/chosen": -1.2676080465316772, | |
| "logps/rejected": -1.5205793380737305, | |
| "loss": 1.3311, | |
| "odds_ratio_loss": 0.6347194910049438, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.1267608106136322, | |
| "rewards/margins": 0.025297129526734352, | |
| "rewards/rejected": -0.1520579308271408, | |
| "sft_loss": 1.2676080465316772, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 1.642675698560542, | |
| "grad_norm": 3.6740529537200928, | |
| "learning_rate": 2.1244588404994648e-06, | |
| "logits/chosen": 251.4237518310547, | |
| "logits/rejected": 251.6488494873047, | |
| "logps/chosen": -1.2188889980316162, | |
| "logps/rejected": -1.379131555557251, | |
| "loss": 1.2882, | |
| "odds_ratio_loss": 0.693446695804596, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.1218889132142067, | |
| "rewards/margins": 0.016024235635995865, | |
| "rewards/rejected": -0.13791313767433167, | |
| "sft_loss": 1.2188889980316162, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 1.65961049957663, | |
| "grad_norm": 6.992788791656494, | |
| "learning_rate": 2.08064853874936e-06, | |
| "logits/chosen": 253.6184539794922, | |
| "logits/rejected": 253.8686981201172, | |
| "logps/chosen": -1.2761471271514893, | |
| "logps/rejected": -1.4780161380767822, | |
| "loss": 1.3423, | |
| "odds_ratio_loss": 0.6611601114273071, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.12761470675468445, | |
| "rewards/margins": 0.020186906680464745, | |
| "rewards/rejected": -0.14780160784721375, | |
| "sft_loss": 1.2761471271514893, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 1.676545300592718, | |
| "grad_norm": 4.474498748779297, | |
| "learning_rate": 2.0369703571452387e-06, | |
| "logits/chosen": 253.08810424804688, | |
| "logits/rejected": 253.7334747314453, | |
| "logps/chosen": -1.1575729846954346, | |
| "logps/rejected": -1.4460541009902954, | |
| "loss": 1.2163, | |
| "odds_ratio_loss": 0.5876318216323853, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.1157573014497757, | |
| "rewards/margins": 0.02884811721742153, | |
| "rewards/rejected": -0.14460542798042297, | |
| "sft_loss": 1.1575729846954346, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 1.6934801016088061, | |
| "grad_norm": 2.3749420642852783, | |
| "learning_rate": 1.993438056859441e-06, | |
| "logits/chosen": 254.9521026611328, | |
| "logits/rejected": 255.0730743408203, | |
| "logps/chosen": -1.2478586435317993, | |
| "logps/rejected": -1.4748018980026245, | |
| "loss": 1.3103, | |
| "odds_ratio_loss": 0.6246139407157898, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.124785877764225, | |
| "rewards/margins": 0.022694315761327744, | |
| "rewards/rejected": -0.14748017489910126, | |
| "sft_loss": 1.2478586435317993, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.6934801016088061, | |
| "eval_logits/chosen": 252.814697265625, | |
| "eval_logits/rejected": 253.1349639892578, | |
| "eval_logps/chosen": -1.2641632556915283, | |
| "eval_logps/rejected": -1.4885586500167847, | |
| "eval_loss": 1.3313392400741577, | |
| "eval_odds_ratio_loss": 0.6717599630355835, | |
| "eval_rewards/accuracies": 0.569523811340332, | |
| "eval_rewards/chosen": -0.12641634047031403, | |
| "eval_rewards/margins": 0.022439539432525635, | |
| "eval_rewards/rejected": -0.14885587990283966, | |
| "eval_runtime": 221.9589, | |
| "eval_samples_per_second": 4.731, | |
| "eval_sft_loss": 1.2641632556915283, | |
| "eval_steps_per_second": 2.365, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.710414902624894, | |
| "grad_norm": 2.523041009902954, | |
| "learning_rate": 1.9500653531031917e-06, | |
| "logits/chosen": 253.83682250976562, | |
| "logits/rejected": 254.0968780517578, | |
| "logps/chosen": -1.231729507446289, | |
| "logps/rejected": -1.4748117923736572, | |
| "loss": 1.2981, | |
| "odds_ratio_loss": 0.6637791395187378, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12317295372486115, | |
| "rewards/margins": 0.02430824190378189, | |
| "rewards/rejected": -0.14748118817806244, | |
| "sft_loss": 1.231729507446289, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 1.7273497036409822, | |
| "grad_norm": 2.989529848098755, | |
| "learning_rate": 1.9068659108055117e-06, | |
| "logits/chosen": 252.50570678710938, | |
| "logits/rejected": 252.9430389404297, | |
| "logps/chosen": -1.2413790225982666, | |
| "logps/rejected": -1.4306474924087524, | |
| "loss": 1.3076, | |
| "odds_ratio_loss": 0.6622155904769897, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12413790076971054, | |
| "rewards/margins": 0.018926845863461494, | |
| "rewards/rejected": -0.14306475222110748, | |
| "sft_loss": 1.2413790225982666, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 1.7442845046570703, | |
| "grad_norm": 1.9568462371826172, | |
| "learning_rate": 1.863853340307962e-06, | |
| "logits/chosen": 252.41305541992188, | |
| "logits/rejected": 253.2246551513672, | |
| "logps/chosen": -1.0862369537353516, | |
| "logps/rejected": -1.4562585353851318, | |
| "loss": 1.1465, | |
| "odds_ratio_loss": 0.6029458045959473, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.1086236983537674, | |
| "rewards/margins": 0.037002164870500565, | |
| "rewards/rejected": -0.14562585949897766, | |
| "sft_loss": 1.0862369537353516, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 1.7612193056731584, | |
| "grad_norm": 2.321819543838501, | |
| "learning_rate": 1.8210411930766019e-06, | |
| "logits/chosen": 253.8018035888672, | |
| "logits/rejected": 253.84048461914062, | |
| "logps/chosen": -1.2366716861724854, | |
| "logps/rejected": -1.473456621170044, | |
| "loss": 1.3013, | |
| "odds_ratio_loss": 0.6460654139518738, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.1236671656370163, | |
| "rewards/margins": 0.023678502067923546, | |
| "rewards/rejected": -0.1473456621170044, | |
| "sft_loss": 1.2366716861724854, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 1.7781541066892466, | |
| "grad_norm": 2.2399725914001465, | |
| "learning_rate": 1.7784429574324803e-06, | |
| "logits/chosen": 253.9780731201172, | |
| "logits/rejected": 253.922607421875, | |
| "logps/chosen": -1.199683427810669, | |
| "logps/rejected": -1.4928243160247803, | |
| "loss": 1.2597, | |
| "odds_ratio_loss": 0.599722683429718, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.11996833980083466, | |
| "rewards/margins": 0.02931409515440464, | |
| "rewards/rejected": -0.14928242564201355, | |
| "sft_loss": 1.199683427810669, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 1.7950889077053345, | |
| "grad_norm": 1.5600874423980713, | |
| "learning_rate": 1.7360720543020327e-06, | |
| "logits/chosen": 254.2810516357422, | |
| "logits/rejected": 254.4838409423828, | |
| "logps/chosen": -1.2215511798858643, | |
| "logps/rejected": -1.3866732120513916, | |
| "loss": 1.2883, | |
| "odds_ratio_loss": 0.6674426794052124, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12215511500835419, | |
| "rewards/margins": 0.016512203961610794, | |
| "rewards/rejected": -0.13866731524467468, | |
| "sft_loss": 1.2215511798858643, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 1.8120237087214224, | |
| "grad_norm": 6.640655994415283, | |
| "learning_rate": 1.6939418329887042e-06, | |
| "logits/chosen": 254.4890594482422, | |
| "logits/rejected": 254.67626953125, | |
| "logps/chosen": -1.2743191719055176, | |
| "logps/rejected": -1.505152940750122, | |
| "loss": 1.3393, | |
| "odds_ratio_loss": 0.6494858264923096, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12743191421031952, | |
| "rewards/margins": 0.023083385080099106, | |
| "rewards/rejected": -0.15051528811454773, | |
| "sft_loss": 1.2743191719055176, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 1.8289585097375105, | |
| "grad_norm": 2.518933057785034, | |
| "learning_rate": 1.6520655669671467e-06, | |
| "logits/chosen": 254.40719604492188, | |
| "logits/rejected": 254.98049926757812, | |
| "logps/chosen": -1.2717182636260986, | |
| "logps/rejected": -1.383216142654419, | |
| "loss": 1.3404, | |
| "odds_ratio_loss": 0.686890721321106, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.1271718293428421, | |
| "rewards/margins": 0.011149783618748188, | |
| "rewards/rejected": -0.13832160830497742, | |
| "sft_loss": 1.2717182636260986, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 1.8458933107535986, | |
| "grad_norm": 2.115421772003174, | |
| "learning_rate": 1.610456449701294e-06, | |
| "logits/chosen": 254.34115600585938, | |
| "logits/rejected": 254.33877563476562, | |
| "logps/chosen": -1.2869365215301514, | |
| "logps/rejected": -1.4801461696624756, | |
| "loss": 1.356, | |
| "odds_ratio_loss": 0.6902877688407898, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12869365513324738, | |
| "rewards/margins": 0.019320974126458168, | |
| "rewards/rejected": -0.1480146199464798, | |
| "sft_loss": 1.2869365215301514, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 1.8628281117696868, | |
| "grad_norm": 3.411358118057251, | |
| "learning_rate": 1.5691275904876545e-06, | |
| "logits/chosen": 253.40380859375, | |
| "logits/rejected": 253.9809112548828, | |
| "logps/chosen": -1.3146874904632568, | |
| "logps/rejected": -1.605054497718811, | |
| "loss": 1.3787, | |
| "odds_ratio_loss": 0.6398962140083313, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.1314687579870224, | |
| "rewards/margins": 0.02903667651116848, | |
| "rewards/rejected": -0.16050544381141663, | |
| "sft_loss": 1.3146874904632568, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 1.879762912785775, | |
| "grad_norm": 2.1282906532287598, | |
| "learning_rate": 1.5280920103251235e-06, | |
| "logits/chosen": 254.62075805664062, | |
| "logits/rejected": 255.1248321533203, | |
| "logps/chosen": -1.219244360923767, | |
| "logps/rejected": -1.4468841552734375, | |
| "loss": 1.2855, | |
| "odds_ratio_loss": 0.6621590852737427, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12192445993423462, | |
| "rewards/margins": 0.022763973101973534, | |
| "rewards/rejected": -0.1446884274482727, | |
| "sft_loss": 1.219244360923767, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 1.8966977138018628, | |
| "grad_norm": 3.646892786026001, | |
| "learning_rate": 1.4873626378126015e-06, | |
| "logits/chosen": 253.64559936523438, | |
| "logits/rejected": 253.7223663330078, | |
| "logps/chosen": -1.2419589757919312, | |
| "logps/rejected": -1.5452362298965454, | |
| "loss": 1.3108, | |
| "odds_ratio_loss": 0.6882845759391785, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.12419591099023819, | |
| "rewards/margins": 0.03032771870493889, | |
| "rewards/rejected": -0.15452361106872559, | |
| "sft_loss": 1.2419589757919312, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 1.913632514817951, | |
| "grad_norm": 2.910693407058716, | |
| "learning_rate": 1.446952305075738e-06, | |
| "logits/chosen": 255.579833984375, | |
| "logits/rejected": 255.78054809570312, | |
| "logps/chosen": -1.220640778541565, | |
| "logps/rejected": -1.3528368473052979, | |
| "loss": 1.2889, | |
| "odds_ratio_loss": 0.6829566359519958, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12206407636404037, | |
| "rewards/margins": 0.013219590298831463, | |
| "rewards/rejected": -0.1352836638689041, | |
| "sft_loss": 1.220640778541565, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 1.9305673158340388, | |
| "grad_norm": 1.925073504447937, | |
| "learning_rate": 1.406873743724065e-06, | |
| "logits/chosen": 254.1175537109375, | |
| "logits/rejected": 253.83895874023438, | |
| "logps/chosen": -1.2488387823104858, | |
| "logps/rejected": -1.625109314918518, | |
| "loss": 1.3124, | |
| "odds_ratio_loss": 0.6353055238723755, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.12488389015197754, | |
| "rewards/margins": 0.03762705251574516, | |
| "rewards/rejected": -0.1625109314918518, | |
| "sft_loss": 1.2488387823104858, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 1.947502116850127, | |
| "grad_norm": 1.9894033670425415, | |
| "learning_rate": 1.3671395808397898e-06, | |
| "logits/chosen": 254.44320678710938, | |
| "logits/rejected": 254.1394500732422, | |
| "logps/chosen": -1.1459629535675049, | |
| "logps/rejected": -1.3053247928619385, | |
| "loss": 1.212, | |
| "odds_ratio_loss": 0.6603742241859436, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.11459628492593765, | |
| "rewards/margins": 0.015936192125082016, | |
| "rewards/rejected": -0.13053248822689056, | |
| "sft_loss": 1.1459629535675049, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 1.964436917866215, | |
| "grad_norm": 8.539448738098145, | |
| "learning_rate": 1.3277623349995418e-06, | |
| "logits/chosen": 254.93515014648438, | |
| "logits/rejected": 256.26190185546875, | |
| "logps/chosen": -1.2024309635162354, | |
| "logps/rejected": -1.3554456233978271, | |
| "loss": 1.2708, | |
| "odds_ratio_loss": 0.6840348243713379, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.12024309486150742, | |
| "rewards/margins": 0.015301482751965523, | |
| "rewards/rejected": -0.1355445832014084, | |
| "sft_loss": 1.2024309635162354, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 1.9813717188823032, | |
| "grad_norm": 3.0065865516662598, | |
| "learning_rate": 1.2887544123302781e-06, | |
| "logits/chosen": 255.7893524169922, | |
| "logits/rejected": 255.7041473388672, | |
| "logps/chosen": -1.2803890705108643, | |
| "logps/rejected": -1.3648675680160522, | |
| "loss": 1.3554, | |
| "odds_ratio_loss": 0.7498189210891724, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.1280389130115509, | |
| "rewards/margins": 0.008447853848338127, | |
| "rewards/rejected": -0.1364867389202118, | |
| "sft_loss": 1.2803890705108643, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 1.9983065198983911, | |
| "grad_norm": 1.9976747035980225, | |
| "learning_rate": 1.2501281026006393e-06, | |
| "logits/chosen": 253.8057403564453, | |
| "logits/rejected": 253.51309204101562, | |
| "logps/chosen": -1.2094168663024902, | |
| "logps/rejected": -1.379926323890686, | |
| "loss": 1.2795, | |
| "odds_ratio_loss": 0.7010444402694702, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.12094169855117798, | |
| "rewards/margins": 0.017050940543413162, | |
| "rewards/rejected": -0.13799263536930084, | |
| "sft_loss": 1.2094168663024902, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 2.015241320914479, | |
| "grad_norm": 2.3396496772766113, | |
| "learning_rate": 1.2118955753489523e-06, | |
| "logits/chosen": 253.90792846679688, | |
| "logits/rejected": 254.0242462158203, | |
| "logps/chosen": -1.2840607166290283, | |
| "logps/rejected": -1.4544405937194824, | |
| "loss": 1.3511, | |
| "odds_ratio_loss": 0.6707578897476196, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.1284060776233673, | |
| "rewards/margins": 0.01703798957169056, | |
| "rewards/rejected": -0.14544406533241272, | |
| "sft_loss": 1.2840607166290283, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 2.032176121930567, | |
| "grad_norm": 3.239753007888794, | |
| "learning_rate": 1.1740688760491189e-06, | |
| "logits/chosen": 253.80667114257812, | |
| "logits/rejected": 254.23452758789062, | |
| "logps/chosen": -1.284844160079956, | |
| "logps/rejected": -1.5715402364730835, | |
| "loss": 1.3467, | |
| "odds_ratio_loss": 0.6180799007415771, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.12848441302776337, | |
| "rewards/margins": 0.02866961434483528, | |
| "rewards/rejected": -0.15715403854846954, | |
| "sft_loss": 1.284844160079956, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 2.0491109229466553, | |
| "grad_norm": 3.396362066268921, | |
| "learning_rate": 1.1366599223155847e-06, | |
| "logits/chosen": 254.4167022705078, | |
| "logits/rejected": 254.499755859375, | |
| "logps/chosen": -1.2031781673431396, | |
| "logps/rejected": -1.4534327983856201, | |
| "loss": 1.2695, | |
| "odds_ratio_loss": 0.66279536485672, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12031783163547516, | |
| "rewards/margins": 0.025025445967912674, | |
| "rewards/rejected": -0.14534327387809753, | |
| "sft_loss": 1.2031781673431396, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 2.0660457239627434, | |
| "grad_norm": 2.935955762863159, | |
| "learning_rate": 1.0996805001486067e-06, | |
| "logits/chosen": 253.8230438232422, | |
| "logits/rejected": 254.1538848876953, | |
| "logps/chosen": -1.2398602962493896, | |
| "logps/rejected": -1.4281715154647827, | |
| "loss": 1.306, | |
| "odds_ratio_loss": 0.6616999506950378, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.12398604303598404, | |
| "rewards/margins": 0.018831118941307068, | |
| "rewards/rejected": -0.1428171545267105, | |
| "sft_loss": 1.2398602962493896, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 2.0829805249788316, | |
| "grad_norm": 5.044885635375977, | |
| "learning_rate": 1.0631422602209608e-06, | |
| "logits/chosen": 253.986083984375, | |
| "logits/rejected": 254.6449737548828, | |
| "logps/chosen": -1.2787022590637207, | |
| "logps/rejected": -1.4583269357681274, | |
| "loss": 1.3487, | |
| "odds_ratio_loss": 0.700279951095581, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12787023186683655, | |
| "rewards/margins": 0.01796245574951172, | |
| "rewards/rejected": -0.14583268761634827, | |
| "sft_loss": 1.2787022590637207, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 2.0999153259949197, | |
| "grad_norm": 1.7120074033737183, | |
| "learning_rate": 1.027056714207319e-06, | |
| "logits/chosen": 255.4936981201172, | |
| "logits/rejected": 255.05712890625, | |
| "logps/chosen": -1.317291498184204, | |
| "logps/rejected": -1.599304437637329, | |
| "loss": 1.3792, | |
| "odds_ratio_loss": 0.6188714504241943, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.1317291557788849, | |
| "rewards/margins": 0.02820129692554474, | |
| "rewards/rejected": -0.15993043780326843, | |
| "sft_loss": 1.317291498184204, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 2.116850127011008, | |
| "grad_norm": 2.1434166431427, | |
| "learning_rate": 9.914352311573838e-07, | |
| "logits/chosen": 254.13870239257812, | |
| "logits/rejected": 253.87496948242188, | |
| "logps/chosen": -1.0639550685882568, | |
| "logps/rejected": -1.3396233320236206, | |
| "loss": 1.1253, | |
| "odds_ratio_loss": 0.6130257248878479, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.10639550536870956, | |
| "rewards/margins": 0.027566824108362198, | |
| "rewards/rejected": -0.13396233320236206, | |
| "sft_loss": 1.0639550685882568, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 2.1337849280270955, | |
| "grad_norm": 1.7394886016845703, | |
| "learning_rate": 9.562890339139877e-07, | |
| "logits/chosen": 252.9702606201172, | |
| "logits/rejected": 253.41909790039062, | |
| "logps/chosen": -1.1834205389022827, | |
| "logps/rejected": -1.4220540523529053, | |
| "loss": 1.2498, | |
| "odds_ratio_loss": 0.6642176508903503, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11834204196929932, | |
| "rewards/margins": 0.02386336401104927, | |
| "rewards/rejected": -0.14220541715621948, | |
| "sft_loss": 1.1834205389022827, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 2.1507197290431836, | |
| "grad_norm": 2.333282232284546, | |
| "learning_rate": 9.216291955772374e-07, | |
| "logits/chosen": 253.9619903564453, | |
| "logits/rejected": 254.09951782226562, | |
| "logps/chosen": -1.1872704029083252, | |
| "logps/rejected": -1.3576862812042236, | |
| "loss": 1.2522, | |
| "odds_ratio_loss": 0.6492857933044434, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.11872704327106476, | |
| "rewards/margins": 0.017041588202118874, | |
| "rewards/rejected": -0.13576863706111908, | |
| "sft_loss": 1.1872704029083252, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 2.167654530059272, | |
| "grad_norm": 6.051000595092773, | |
| "learning_rate": 8.874666360158457e-07, | |
| "logits/chosen": 253.09268188476562, | |
| "logits/rejected": 253.14846801757812, | |
| "logps/chosen": -1.1666978597640991, | |
| "logps/rejected": -1.3771264553070068, | |
| "loss": 1.2333, | |
| "odds_ratio_loss": 0.6663509607315063, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.11666979640722275, | |
| "rewards/margins": 0.021042857319116592, | |
| "rewards/rejected": -0.13771264255046844, | |
| "sft_loss": 1.1666978597640991, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 2.18458933107536, | |
| "grad_norm": 2.681307315826416, | |
| "learning_rate": 8.538121184267315e-07, | |
| "logits/chosen": 252.548095703125, | |
| "logits/rejected": 253.38436889648438, | |
| "logps/chosen": -1.107893705368042, | |
| "logps/rejected": -1.3506486415863037, | |
| "loss": 1.1702, | |
| "odds_ratio_loss": 0.6233555674552917, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11078937351703644, | |
| "rewards/margins": 0.024275511503219604, | |
| "rewards/rejected": -0.13506487011909485, | |
| "sft_loss": 1.107893705368042, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 2.201524132091448, | |
| "grad_norm": 2.337139368057251, | |
| "learning_rate": 8.206762459439907e-07, | |
| "logits/chosen": 252.81088256835938, | |
| "logits/rejected": 253.55819702148438, | |
| "logps/chosen": -1.2137644290924072, | |
| "logps/rejected": -1.405731201171875, | |
| "loss": 1.2819, | |
| "odds_ratio_loss": 0.6810044050216675, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12137645483016968, | |
| "rewards/margins": 0.019196683540940285, | |
| "rewards/rejected": -0.14057312905788422, | |
| "sft_loss": 1.2137644290924072, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 2.218458933107536, | |
| "grad_norm": 2.8654181957244873, | |
| "learning_rate": 7.880694582982898e-07, | |
| "logits/chosen": 254.5794677734375, | |
| "logits/rejected": 255.022705078125, | |
| "logps/chosen": -1.2548917531967163, | |
| "logps/rejected": -1.4934711456298828, | |
| "loss": 1.3177, | |
| "odds_ratio_loss": 0.6284439563751221, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12548920512199402, | |
| "rewards/margins": 0.02385791949927807, | |
| "rewards/rejected": -0.14934709668159485, | |
| "sft_loss": 1.2548917531967163, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 2.235393734123624, | |
| "grad_norm": 2.6947178840637207, | |
| "learning_rate": 7.560020285277401e-07, | |
| "logits/chosen": 254.29129028320312, | |
| "logits/rejected": 254.16165161132812, | |
| "logps/chosen": -1.2197682857513428, | |
| "logps/rejected": -1.3262290954589844, | |
| "loss": 1.2905, | |
| "odds_ratio_loss": 0.7075908780097961, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.1219768151640892, | |
| "rewards/margins": 0.010646088980138302, | |
| "rewards/rejected": -0.13262291252613068, | |
| "sft_loss": 1.2197682857513428, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 2.252328535139712, | |
| "grad_norm": 3.4201550483703613, | |
| "learning_rate": 7.244840597412956e-07, | |
| "logits/chosen": 253.959228515625, | |
| "logits/rejected": 254.095458984375, | |
| "logps/chosen": -1.244969129562378, | |
| "logps/rejected": -1.3216217756271362, | |
| "loss": 1.3196, | |
| "odds_ratio_loss": 0.7459138631820679, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.12449691444635391, | |
| "rewards/margins": 0.007665269076824188, | |
| "rewards/rejected": -0.1321621835231781, | |
| "sft_loss": 1.244969129562378, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 2.2692633361558, | |
| "grad_norm": 4.195887565612793, | |
| "learning_rate": 6.935254819356796e-07, | |
| "logits/chosen": 255.4742889404297, | |
| "logits/rejected": 255.7603759765625, | |
| "logps/chosen": -1.221064805984497, | |
| "logps/rejected": -1.4360755681991577, | |
| "loss": 1.2872, | |
| "odds_ratio_loss": 0.6609319448471069, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.12210647761821747, | |
| "rewards/margins": 0.021501056849956512, | |
| "rewards/rejected": -0.14360754191875458, | |
| "sft_loss": 1.221064805984497, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 2.2861981371718882, | |
| "grad_norm": 1.8310470581054688, | |
| "learning_rate": 6.631360488668662e-07, | |
| "logits/chosen": 253.05575561523438, | |
| "logits/rejected": 253.70724487304688, | |
| "logps/chosen": -1.1173431873321533, | |
| "logps/rejected": -1.3836314678192139, | |
| "loss": 1.1808, | |
| "odds_ratio_loss": 0.634362518787384, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.11173433065414429, | |
| "rewards/margins": 0.02662881650030613, | |
| "rewards/rejected": -0.13836315274238586, | |
| "sft_loss": 1.1173431873321533, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 2.3031329381879764, | |
| "grad_norm": 3.754059076309204, | |
| "learning_rate": 6.333253349770672e-07, | |
| "logits/chosen": 255.578125, | |
| "logits/rejected": 255.3465118408203, | |
| "logps/chosen": -1.257102370262146, | |
| "logps/rejected": -1.4100966453552246, | |
| "loss": 1.3274, | |
| "odds_ratio_loss": 0.7030817270278931, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12571023404598236, | |
| "rewards/margins": 0.015299415215849876, | |
| "rewards/rejected": -0.14100965857505798, | |
| "sft_loss": 1.257102370262146, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 2.3200677392040645, | |
| "grad_norm": 1.6480987071990967, | |
| "learning_rate": 6.041027323782364e-07, | |
| "logits/chosen": 254.044921875, | |
| "logits/rejected": 254.229248046875, | |
| "logps/chosen": -1.1556494235992432, | |
| "logps/rejected": -1.4046926498413086, | |
| "loss": 1.2175, | |
| "odds_ratio_loss": 0.6188673973083496, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.11556495726108551, | |
| "rewards/margins": 0.02490430511534214, | |
| "rewards/rejected": -0.1404692679643631, | |
| "sft_loss": 1.1556494235992432, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 2.337002540220152, | |
| "grad_norm": 2.606630563735962, | |
| "learning_rate": 5.754774478929969e-07, | |
| "logits/chosen": 253.4460906982422, | |
| "logits/rejected": 253.7098388671875, | |
| "logps/chosen": -1.2188177108764648, | |
| "logps/rejected": -1.42665433883667, | |
| "loss": 1.2822, | |
| "odds_ratio_loss": 0.6334772706031799, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12188176810741425, | |
| "rewards/margins": 0.020783668383955956, | |
| "rewards/rejected": -0.14266543090343475, | |
| "sft_loss": 1.2188177108764648, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 2.3539373412362403, | |
| "grad_norm": 4.384988784790039, | |
| "learning_rate": 5.474585001539634e-07, | |
| "logits/chosen": 254.74624633789062, | |
| "logits/rejected": 255.39364624023438, | |
| "logps/chosen": -1.1506468057632446, | |
| "logps/rejected": -1.3720335960388184, | |
| "loss": 1.2119, | |
| "odds_ratio_loss": 0.612425684928894, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.11506466567516327, | |
| "rewards/margins": 0.02213868871331215, | |
| "rewards/rejected": -0.1372033655643463, | |
| "sft_loss": 1.1506468057632446, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 2.3708721422523285, | |
| "grad_norm": 1.8550916910171509, | |
| "learning_rate": 5.200547167623424e-07, | |
| "logits/chosen": 255.02920532226562, | |
| "logits/rejected": 254.79867553710938, | |
| "logps/chosen": -1.2350322008132935, | |
| "logps/rejected": -1.4950242042541504, | |
| "loss": 1.2975, | |
| "odds_ratio_loss": 0.6242570281028748, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.12350322306156158, | |
| "rewards/margins": 0.02599920891225338, | |
| "rewards/rejected": -0.14950242638587952, | |
| "sft_loss": 1.2350322008132935, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 2.3878069432684166, | |
| "grad_norm": 1.3218871355056763, | |
| "learning_rate": 4.932747315067271e-07, | |
| "logits/chosen": 253.67959594726562, | |
| "logits/rejected": 253.9455108642578, | |
| "logps/chosen": -1.1884933710098267, | |
| "logps/rejected": -1.4974465370178223, | |
| "loss": 1.2494, | |
| "odds_ratio_loss": 0.6091898083686829, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.11884935200214386, | |
| "rewards/margins": 0.030895307660102844, | |
| "rewards/rejected": -0.1497446447610855, | |
| "sft_loss": 1.1884933710098267, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 2.4047417442845047, | |
| "grad_norm": 4.312244415283203, | |
| "learning_rate": 4.6712698164294553e-07, | |
| "logits/chosen": 253.365966796875, | |
| "logits/rejected": 254.00637817382812, | |
| "logps/chosen": -1.1741528511047363, | |
| "logps/rejected": -1.393843650817871, | |
| "loss": 1.2374, | |
| "odds_ratio_loss": 0.6321112513542175, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.11741527169942856, | |
| "rewards/margins": 0.021969076246023178, | |
| "rewards/rejected": -0.13938435912132263, | |
| "sft_loss": 1.1741528511047363, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 2.421676545300593, | |
| "grad_norm": 2.516547918319702, | |
| "learning_rate": 4.41619705235842e-07, | |
| "logits/chosen": 254.66812133789062, | |
| "logits/rejected": 255.44058227539062, | |
| "logps/chosen": -1.243293285369873, | |
| "logps/rejected": -1.6239792108535767, | |
| "loss": 1.3027, | |
| "odds_ratio_loss": 0.5943632125854492, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.1243293434381485, | |
| "rewards/margins": 0.03806859627366066, | |
| "rewards/rejected": -0.16239793598651886, | |
| "sft_loss": 1.243293285369873, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 2.438611346316681, | |
| "grad_norm": 1.4834553003311157, | |
| "learning_rate": 4.167609385637961e-07, | |
| "logits/chosen": 254.78579711914062, | |
| "logits/rejected": 255.18881225585938, | |
| "logps/chosen": -1.2608935832977295, | |
| "logps/rejected": -1.425330638885498, | |
| "loss": 1.3255, | |
| "odds_ratio_loss": 0.6463701128959656, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12608936429023743, | |
| "rewards/margins": 0.016443699598312378, | |
| "rewards/rejected": -0.1425330489873886, | |
| "sft_loss": 1.2608935832977295, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 2.4555461473327687, | |
| "grad_norm": 3.3783388137817383, | |
| "learning_rate": 3.9255851358683567e-07, | |
| "logits/chosen": 254.845458984375, | |
| "logits/rejected": 255.02957153320312, | |
| "logps/chosen": -1.1708389520645142, | |
| "logps/rejected": -1.3822017908096313, | |
| "loss": 1.2391, | |
| "odds_ratio_loss": 0.6828811764717102, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.11708390712738037, | |
| "rewards/margins": 0.02113628014922142, | |
| "rewards/rejected": -0.1382201761007309, | |
| "sft_loss": 1.1708389520645142, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 2.472480948348857, | |
| "grad_norm": 3.0601887702941895, | |
| "learning_rate": 3.690200554791082e-07, | |
| "logits/chosen": 252.8287811279297, | |
| "logits/rejected": 252.99655151367188, | |
| "logps/chosen": -1.207948923110962, | |
| "logps/rejected": -1.4645483493804932, | |
| "loss": 1.2682, | |
| "odds_ratio_loss": 0.6026018261909485, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12079490721225739, | |
| "rewards/margins": 0.025659915059804916, | |
| "rewards/rejected": -0.1464548110961914, | |
| "sft_loss": 1.207948923110962, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 2.489415749364945, | |
| "grad_norm": 4.717799186706543, | |
| "learning_rate": 3.461529802265079e-07, | |
| "logits/chosen": 253.954833984375, | |
| "logits/rejected": 254.8602752685547, | |
| "logps/chosen": -1.2261638641357422, | |
| "logps/rejected": -1.3805973529815674, | |
| "loss": 1.2931, | |
| "odds_ratio_loss": 0.6691168546676636, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.12261638790369034, | |
| "rewards/margins": 0.01544335950165987, | |
| "rewards/rejected": -0.13805975019931793, | |
| "sft_loss": 1.2261638641357422, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 2.506350550381033, | |
| "grad_norm": 2.0962188243865967, | |
| "learning_rate": 3.2396449229020883e-07, | |
| "logits/chosen": 254.4154815673828, | |
| "logits/rejected": 254.2891082763672, | |
| "logps/chosen": -1.242130994796753, | |
| "logps/rejected": -1.4029905796051025, | |
| "loss": 1.3096, | |
| "odds_ratio_loss": 0.6747825741767883, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.1242130845785141, | |
| "rewards/margins": 0.016085978597402573, | |
| "rewards/rejected": -0.14029906690120697, | |
| "sft_loss": 1.242130994796753, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 2.523285351397121, | |
| "grad_norm": 4.5739922523498535, | |
| "learning_rate": 3.024615823368371e-07, | |
| "logits/chosen": 253.8447723388672, | |
| "logits/rejected": 253.74185180664062, | |
| "logps/chosen": -1.1965720653533936, | |
| "logps/rejected": -1.408341407775879, | |
| "loss": 1.2608, | |
| "odds_ratio_loss": 0.6427493691444397, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.11965718120336533, | |
| "rewards/margins": 0.021176962181925774, | |
| "rewards/rejected": -0.14083415269851685, | |
| "sft_loss": 1.1965720653533936, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 2.5402201524132093, | |
| "grad_norm": 2.1523611545562744, | |
| "learning_rate": 2.8165102503600716e-07, | |
| "logits/chosen": 255.3013153076172, | |
| "logits/rejected": 254.82162475585938, | |
| "logps/chosen": -1.1430598497390747, | |
| "logps/rejected": -1.4227603673934937, | |
| "loss": 1.2057, | |
| "odds_ratio_loss": 0.6268513798713684, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.11430598795413971, | |
| "rewards/margins": 0.027970051392912865, | |
| "rewards/rejected": -0.14227603375911713, | |
| "sft_loss": 1.1430598497390747, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.5402201524132093, | |
| "eval_logits/chosen": 253.5438995361328, | |
| "eval_logits/rejected": 253.86448669433594, | |
| "eval_logps/chosen": -1.2487919330596924, | |
| "eval_logps/rejected": -1.470909833908081, | |
| "eval_loss": 1.3159173727035522, | |
| "eval_odds_ratio_loss": 0.6712530851364136, | |
| "eval_rewards/accuracies": 0.561904788017273, | |
| "eval_rewards/chosen": -0.12487921118736267, | |
| "eval_rewards/margins": 0.02221176214516163, | |
| "eval_rewards/rejected": -0.14709095656871796, | |
| "eval_runtime": 222.3485, | |
| "eval_samples_per_second": 4.722, | |
| "eval_sft_loss": 1.2487919330596924, | |
| "eval_steps_per_second": 2.361, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.557154953429297, | |
| "grad_norm": 1.7336281538009644, | |
| "learning_rate": 2.615393769259039e-07, | |
| "logits/chosen": 254.2406768798828, | |
| "logits/rejected": 254.7086639404297, | |
| "logps/chosen": -1.319853663444519, | |
| "logps/rejected": -1.414665937423706, | |
| "loss": 1.3937, | |
| "odds_ratio_loss": 0.7381461262702942, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.1319853961467743, | |
| "rewards/margins": 0.009481215849518776, | |
| "rewards/rejected": -0.14146658778190613, | |
| "sft_loss": 1.319853663444519, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 2.574089754445385, | |
| "grad_norm": 2.717740297317505, | |
| "learning_rate": 2.421329743475917e-07, | |
| "logits/chosen": 254.42990112304688, | |
| "logits/rejected": 254.5959014892578, | |
| "logps/chosen": -1.1657871007919312, | |
| "logps/rejected": -1.3640342950820923, | |
| "loss": 1.2334, | |
| "odds_ratio_loss": 0.6763022541999817, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.11657872051000595, | |
| "rewards/margins": 0.019824711605906487, | |
| "rewards/rejected": -0.13640344142913818, | |
| "sft_loss": 1.1657871007919312, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 2.5910245554614733, | |
| "grad_norm": 2.8658461570739746, | |
| "learning_rate": 2.234379314486973e-07, | |
| "logits/chosen": 253.9791259765625, | |
| "logits/rejected": 254.4918670654297, | |
| "logps/chosen": -1.2597442865371704, | |
| "logps/rejected": -1.434729814529419, | |
| "loss": 1.3281, | |
| "odds_ratio_loss": 0.6833871006965637, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12597441673278809, | |
| "rewards/margins": 0.017498571425676346, | |
| "rewards/rejected": -0.14347299933433533, | |
| "sft_loss": 1.2597442865371704, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 2.6079593564775614, | |
| "grad_norm": 1.626412034034729, | |
| "learning_rate": 2.0546013825709783e-07, | |
| "logits/chosen": 254.2110595703125, | |
| "logits/rejected": 254.682373046875, | |
| "logps/chosen": -1.2033625841140747, | |
| "logps/rejected": -1.5869375467300415, | |
| "loss": 1.2628, | |
| "odds_ratio_loss": 0.5944391489028931, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.12033627182245255, | |
| "rewards/margins": 0.03835749253630638, | |
| "rewards/rejected": -0.15869374573230743, | |
| "sft_loss": 1.2033625841140747, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 2.6248941574936495, | |
| "grad_norm": 3.6966705322265625, | |
| "learning_rate": 1.88205258825217e-07, | |
| "logits/chosen": 253.447998046875, | |
| "logits/rejected": 254.12582397460938, | |
| "logps/chosen": -1.0821640491485596, | |
| "logps/rejected": -1.3699367046356201, | |
| "loss": 1.1414, | |
| "odds_ratio_loss": 0.5919198393821716, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.10821641981601715, | |
| "rewards/margins": 0.028777271509170532, | |
| "rewards/rejected": -0.1369936764240265, | |
| "sft_loss": 1.0821640491485596, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 2.6418289585097376, | |
| "grad_norm": 2.906128168106079, | |
| "learning_rate": 1.7167872944552245e-07, | |
| "logits/chosen": 253.732666015625, | |
| "logits/rejected": 254.1513214111328, | |
| "logps/chosen": -1.1799228191375732, | |
| "logps/rejected": -1.3946034908294678, | |
| "loss": 1.2452, | |
| "odds_ratio_loss": 0.6522972583770752, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11799228191375732, | |
| "rewards/margins": 0.021468069404363632, | |
| "rewards/rejected": -0.13946035504341125, | |
| "sft_loss": 1.1799228191375732, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 2.6587637595258258, | |
| "grad_norm": 2.2223029136657715, | |
| "learning_rate": 1.5588575693777142e-07, | |
| "logits/chosen": 255.2970428466797, | |
| "logits/rejected": 255.1865692138672, | |
| "logps/chosen": -1.1618893146514893, | |
| "logps/rejected": -1.3099621534347534, | |
| "loss": 1.2281, | |
| "odds_ratio_loss": 0.6616276502609253, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.11618892103433609, | |
| "rewards/margins": 0.01480729877948761, | |
| "rewards/rejected": -0.1309962272644043, | |
| "sft_loss": 1.1618893146514893, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 2.675698560541914, | |
| "grad_norm": 3.071002721786499, | |
| "learning_rate": 1.4083131700856428e-07, | |
| "logits/chosen": 252.95462036132812, | |
| "logits/rejected": 253.45449829101562, | |
| "logps/chosen": -1.275651454925537, | |
| "logps/rejected": -1.421471357345581, | |
| "loss": 1.3435, | |
| "odds_ratio_loss": 0.6788088083267212, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.1275651454925537, | |
| "rewards/margins": 0.01458200253546238, | |
| "rewards/rejected": -0.14214713871479034, | |
| "sft_loss": 1.275651454925537, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 2.6926333615580016, | |
| "grad_norm": 3.7156240940093994, | |
| "learning_rate": 1.2652015268370315e-07, | |
| "logits/chosen": 255.25674438476562, | |
| "logits/rejected": 255.61105346679688, | |
| "logps/chosen": -1.173892617225647, | |
| "logps/rejected": -1.431540846824646, | |
| "loss": 1.2371, | |
| "odds_ratio_loss": 0.6318042874336243, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.1173892617225647, | |
| "rewards/margins": 0.02576482854783535, | |
| "rewards/rejected": -0.1431540995836258, | |
| "sft_loss": 1.173892617225647, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 2.7095681625740897, | |
| "grad_norm": 3.5745534896850586, | |
| "learning_rate": 1.1295677281386502e-07, | |
| "logits/chosen": 254.6566162109375, | |
| "logits/rejected": 254.93807983398438, | |
| "logps/chosen": -1.3123610019683838, | |
| "logps/rejected": -1.6006231307983398, | |
| "loss": 1.3777, | |
| "odds_ratio_loss": 0.6533895134925842, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.13123610615730286, | |
| "rewards/margins": 0.028826218098402023, | |
| "rewards/rejected": -0.16006234288215637, | |
| "sft_loss": 1.3123610019683838, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 2.726502963590178, | |
| "grad_norm": 2.5153346061706543, | |
| "learning_rate": 1.0014545065404973e-07, | |
| "logits/chosen": 254.83837890625, | |
| "logits/rejected": 255.0562286376953, | |
| "logps/chosen": -1.2541964054107666, | |
| "logps/rejected": -1.501185655593872, | |
| "loss": 1.323, | |
| "odds_ratio_loss": 0.6881270408630371, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12541964650154114, | |
| "rewards/margins": 0.02469891868531704, | |
| "rewards/rejected": -0.15011855959892273, | |
| "sft_loss": 1.2541964054107666, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 2.743437764606266, | |
| "grad_norm": 1.9238418340682983, | |
| "learning_rate": 8.809022251725502e-08, | |
| "logits/chosen": 255.2646484375, | |
| "logits/rejected": 255.28665161132812, | |
| "logps/chosen": -1.2217843532562256, | |
| "logps/rejected": -1.5161769390106201, | |
| "loss": 1.285, | |
| "odds_ratio_loss": 0.6321364045143127, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12217842042446136, | |
| "rewards/margins": 0.029439279809594154, | |
| "rewards/rejected": -0.15161770582199097, | |
| "sft_loss": 1.2217843532562256, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 2.7603725656223537, | |
| "grad_norm": 2.2195630073547363, | |
| "learning_rate": 7.679488650280509e-08, | |
| "logits/chosen": 256.6839904785156, | |
| "logits/rejected": 256.9764404296875, | |
| "logps/chosen": -1.246552586555481, | |
| "logps/rejected": -1.428499460220337, | |
| "loss": 1.3117, | |
| "odds_ratio_loss": 0.6510922312736511, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12465526908636093, | |
| "rewards/margins": 0.01819469965994358, | |
| "rewards/rejected": -0.14284996688365936, | |
| "sft_loss": 1.246552586555481, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 2.777307366638442, | |
| "grad_norm": 5.003395080566406, | |
| "learning_rate": 6.626300129972563e-08, | |
| "logits/chosen": 254.61721801757812, | |
| "logits/rejected": 255.3614044189453, | |
| "logps/chosen": -1.1782416105270386, | |
| "logps/rejected": -1.39430570602417, | |
| "loss": 1.2449, | |
| "odds_ratio_loss": 0.6665213704109192, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.11782417446374893, | |
| "rewards/margins": 0.021606411784887314, | |
| "rewards/rejected": -0.13943056762218475, | |
| "sft_loss": 1.1782416105270386, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 2.79424216765453, | |
| "grad_norm": 2.193207263946533, | |
| "learning_rate": 5.649788506555065e-08, | |
| "logits/chosen": 253.5979461669922, | |
| "logits/rejected": 254.72201538085938, | |
| "logps/chosen": -1.2255361080169678, | |
| "logps/rejected": -1.4795286655426025, | |
| "loss": 1.2878, | |
| "odds_ratio_loss": 0.6227026581764221, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.12255360931158066, | |
| "rewards/margins": 0.02539927139878273, | |
| "rewards/rejected": -0.1479528844356537, | |
| "sft_loss": 1.2255361080169678, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 2.811176968670618, | |
| "grad_norm": 2.1340768337249756, | |
| "learning_rate": 4.7502614380908474e-08, | |
| "logits/chosen": 255.0865020751953, | |
| "logits/rejected": 255.60958862304688, | |
| "logps/chosen": -1.1954472064971924, | |
| "logps/rejected": -1.39599609375, | |
| "loss": 1.2631, | |
| "odds_ratio_loss": 0.6766428351402283, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.11954472959041595, | |
| "rewards/margins": 0.020054878666996956, | |
| "rewards/rejected": -0.13959960639476776, | |
| "sft_loss": 1.1954472064971924, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 2.828111769686706, | |
| "grad_norm": 2.8715949058532715, | |
| "learning_rate": 3.9280023280222066e-08, | |
| "logits/chosen": 254.49807739257812, | |
| "logits/rejected": 254.8501434326172, | |
| "logps/chosen": -1.1896106004714966, | |
| "logps/rejected": -1.3948595523834229, | |
| "loss": 1.2531, | |
| "odds_ratio_loss": 0.6348463892936707, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11896105855703354, | |
| "rewards/margins": 0.020524896681308746, | |
| "rewards/rejected": -0.1394859403371811, | |
| "sft_loss": 1.1896106004714966, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 2.8450465707027943, | |
| "grad_norm": 1.6025958061218262, | |
| "learning_rate": 3.1832702358818855e-08, | |
| "logits/chosen": 254.23095703125, | |
| "logits/rejected": 254.564208984375, | |
| "logps/chosen": -1.2220954895019531, | |
| "logps/rejected": -1.390905737876892, | |
| "loss": 1.288, | |
| "odds_ratio_loss": 0.6586533784866333, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.12220954895019531, | |
| "rewards/margins": 0.016881009563803673, | |
| "rewards/rejected": -0.13909055292606354, | |
| "sft_loss": 1.2220954895019531, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 2.8619813717188824, | |
| "grad_norm": 2.6114532947540283, | |
| "learning_rate": 2.5162997956746647e-08, | |
| "logits/chosen": 254.39810180664062, | |
| "logits/rejected": 254.410400390625, | |
| "logps/chosen": -1.2105869054794312, | |
| "logps/rejected": -1.523001790046692, | |
| "loss": 1.2743, | |
| "odds_ratio_loss": 0.6369892954826355, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12105868011713028, | |
| "rewards/margins": 0.031241485849022865, | |
| "rewards/rejected": -0.15230019390583038, | |
| "sft_loss": 1.2105869054794312, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 2.8789161727349706, | |
| "grad_norm": 5.513568878173828, | |
| "learning_rate": 1.9273011419536914e-08, | |
| "logits/chosen": 254.4501495361328, | |
| "logits/rejected": 254.602783203125, | |
| "logps/chosen": -1.2663971185684204, | |
| "logps/rejected": -1.3631950616836548, | |
| "loss": 1.3412, | |
| "odds_ratio_loss": 0.7475703954696655, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.1266397088766098, | |
| "rewards/margins": 0.009679802693426609, | |
| "rewards/rejected": -0.13631951808929443, | |
| "sft_loss": 1.2663971185684204, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 2.8958509737510583, | |
| "grad_norm": 2.4077532291412354, | |
| "learning_rate": 1.4164598436159083e-08, | |
| "logits/chosen": 254.43917846679688, | |
| "logits/rejected": 254.9999237060547, | |
| "logps/chosen": -1.2015140056610107, | |
| "logps/rejected": -1.3471964597702026, | |
| "loss": 1.2679, | |
| "odds_ratio_loss": 0.6641986966133118, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.12015138566493988, | |
| "rewards/margins": 0.014568252488970757, | |
| "rewards/rejected": -0.1347196400165558, | |
| "sft_loss": 1.2015140056610107, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 2.9127857747671464, | |
| "grad_norm": 1.6812494993209839, | |
| "learning_rate": 9.839368454371556e-09, | |
| "logits/chosen": 253.5565185546875, | |
| "logits/rejected": 253.43716430664062, | |
| "logps/chosen": -1.1501060724258423, | |
| "logps/rejected": -1.454131007194519, | |
| "loss": 1.213, | |
| "odds_ratio_loss": 0.6291440725326538, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.11501061916351318, | |
| "rewards/margins": 0.03040250577032566, | |
| "rewards/rejected": -0.1454131305217743, | |
| "sft_loss": 1.1501060724258423, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 2.9297205757832345, | |
| "grad_norm": 1.9758036136627197, | |
| "learning_rate": 6.298684173650649e-09, | |
| "logits/chosen": 255.69775390625, | |
| "logits/rejected": 255.80801391601562, | |
| "logps/chosen": -1.1473584175109863, | |
| "logps/rejected": -1.4585325717926025, | |
| "loss": 1.2129, | |
| "odds_ratio_loss": 0.6559008359909058, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.11473584175109863, | |
| "rewards/margins": 0.031117429956793785, | |
| "rewards/rejected": -0.14585328102111816, | |
| "sft_loss": 1.1473584175109863, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 2.9466553767993227, | |
| "grad_norm": 1.9825971126556396, | |
| "learning_rate": 3.543661115860686e-09, | |
| "logits/chosen": 253.49093627929688, | |
| "logits/rejected": 254.39404296875, | |
| "logps/chosen": -1.2223292589187622, | |
| "logps/rejected": -1.468933343887329, | |
| "loss": 1.2883, | |
| "odds_ratio_loss": 0.660078763961792, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12223292887210846, | |
| "rewards/margins": 0.02466042898595333, | |
| "rewards/rejected": -0.14689335227012634, | |
| "sft_loss": 1.2223292589187622, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 2.963590177815411, | |
| "grad_norm": 2.3436944484710693, | |
| "learning_rate": 1.575167273800693e-09, | |
| "logits/chosen": 253.98019409179688, | |
| "logits/rejected": 254.1809844970703, | |
| "logps/chosen": -1.1822068691253662, | |
| "logps/rejected": -1.3398927450180054, | |
| "loss": 1.249, | |
| "odds_ratio_loss": 0.6677161455154419, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.11822070181369781, | |
| "rewards/margins": 0.015768591314554214, | |
| "rewards/rejected": -0.13398927450180054, | |
| "sft_loss": 1.1822068691253662, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 2.9805249788314985, | |
| "grad_norm": 6.531856536865234, | |
| "learning_rate": 3.9382283773564676e-10, | |
| "logits/chosen": 254.8570098876953, | |
| "logits/rejected": 255.1466064453125, | |
| "logps/chosen": -1.272533655166626, | |
| "logps/rejected": -1.498098611831665, | |
| "loss": 1.3394, | |
| "odds_ratio_loss": 0.6682497262954712, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12725338339805603, | |
| "rewards/margins": 0.02255650982260704, | |
| "rewards/rejected": -0.14980986714363098, | |
| "sft_loss": 1.272533655166626, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 2.9974597798475866, | |
| "grad_norm": 10.32663345336914, | |
| "learning_rate": 0.0, | |
| "logits/chosen": 253.9761199951172, | |
| "logits/rejected": 253.9351806640625, | |
| "logps/chosen": -1.3296254873275757, | |
| "logps/rejected": -1.604189157485962, | |
| "loss": 1.3966, | |
| "odds_ratio_loss": 0.6696831583976746, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.13296253979206085, | |
| "rewards/margins": 0.02745637856423855, | |
| "rewards/rejected": -0.16041892766952515, | |
| "sft_loss": 1.3296254873275757, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 2.9974597798475866, | |
| "step": 1770, | |
| "total_flos": 2.135641671600046e+18, | |
| "train_loss": 1.3953795637788071, | |
| "train_runtime": 23493.5825, | |
| "train_samples_per_second": 1.206, | |
| "train_steps_per_second": 0.075 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1770, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "total_flos": 2.135641671600046e+18, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |