|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 1.0, |
|
"eval_steps": 500, |
|
"global_step": 625, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 7.936507936507937e-08, |
|
"logits/chosen": 0.16854308545589447, |
|
"logits/rejected": 0.17400208115577698, |
|
"logps/chosen": -327.90460205078125, |
|
"logps/rejected": -253.07960510253906, |
|
"loss": 0.4067, |
|
"rewards/accuracies": 0.0, |
|
"rewards/chosen": 0.0, |
|
"rewards/margins": 0.0, |
|
"rewards/rejected": 0.0, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 7.936507936507937e-07, |
|
"logits/chosen": 0.08604434132575989, |
|
"logits/rejected": 0.1809283345937729, |
|
"logps/chosen": -327.1376647949219, |
|
"logps/rejected": -306.0588684082031, |
|
"loss": 0.3864, |
|
"rewards/accuracies": 0.375, |
|
"rewards/chosen": -5.549057823373005e-05, |
|
"rewards/margins": -0.0001069721911335364, |
|
"rewards/rejected": 5.148161653778516e-05, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 1.5873015873015873e-06, |
|
"logits/chosen": 0.08089748024940491, |
|
"logits/rejected": 0.2491210699081421, |
|
"logps/chosen": -362.65435791015625, |
|
"logps/rejected": -352.3981018066406, |
|
"loss": 0.3729, |
|
"rewards/accuracies": 0.48124998807907104, |
|
"rewards/chosen": 2.957274773507379e-05, |
|
"rewards/margins": 5.038862582296133e-05, |
|
"rewards/rejected": -2.0815870811929926e-05, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 2.380952380952381e-06, |
|
"logits/chosen": 0.18664827942848206, |
|
"logits/rejected": 0.14790232479572296, |
|
"logps/chosen": -312.6676940917969, |
|
"logps/rejected": -324.85516357421875, |
|
"loss": 0.3789, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": 8.431552851106972e-05, |
|
"rewards/margins": 0.00019130071450490505, |
|
"rewards/rejected": -0.00010698519326979294, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 3.1746031746031746e-06, |
|
"logits/chosen": 0.08645667880773544, |
|
"logits/rejected": 0.2045074999332428, |
|
"logps/chosen": -347.3286437988281, |
|
"logps/rejected": -375.4449768066406, |
|
"loss": 0.3765, |
|
"rewards/accuracies": 0.512499988079071, |
|
"rewards/chosen": -0.0001862197823356837, |
|
"rewards/margins": 6.710310117341578e-05, |
|
"rewards/rejected": -0.000253322854405269, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 3.968253968253968e-06, |
|
"logits/chosen": 0.11011010408401489, |
|
"logits/rejected": 0.1821037232875824, |
|
"logps/chosen": -385.66693115234375, |
|
"logps/rejected": -349.8896789550781, |
|
"loss": 0.3738, |
|
"rewards/accuracies": 0.53125, |
|
"rewards/chosen": 4.727130362880416e-05, |
|
"rewards/margins": 0.00021036244288552552, |
|
"rewards/rejected": -0.00016309114289470017, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 4.761904761904762e-06, |
|
"logits/chosen": 0.07650097459554672, |
|
"logits/rejected": 0.177745059132576, |
|
"logps/chosen": -343.5960998535156, |
|
"logps/rejected": -331.03912353515625, |
|
"loss": 0.3796, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": 0.0009819746483117342, |
|
"rewards/margins": 0.0015515821287408471, |
|
"rewards/rejected": -0.0005696073058061302, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 4.998086282661188e-06, |
|
"logits/chosen": 0.11013742536306381, |
|
"logits/rejected": 0.16680167615413666, |
|
"logps/chosen": -361.571044921875, |
|
"logps/rejected": -318.3884582519531, |
|
"loss": 0.3614, |
|
"rewards/accuracies": 0.668749988079071, |
|
"rewards/chosen": 0.0014994305092841387, |
|
"rewards/margins": 0.0018448818009346724, |
|
"rewards/rejected": -0.00034545123344287276, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 4.988720025682995e-06, |
|
"logits/chosen": 0.1778658628463745, |
|
"logits/rejected": 0.19966575503349304, |
|
"logps/chosen": -292.3515625, |
|
"logps/rejected": -321.511962890625, |
|
"loss": 0.3511, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": 0.0031275060027837753, |
|
"rewards/margins": 0.0046959021128714085, |
|
"rewards/rejected": -0.0015683959936723113, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 4.9715789537359126e-06, |
|
"logits/chosen": 0.14186400175094604, |
|
"logits/rejected": 0.18624775111675262, |
|
"logps/chosen": -360.6056823730469, |
|
"logps/rejected": -317.9211730957031, |
|
"loss": 0.3683, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.004151093773543835, |
|
"rewards/margins": 0.006570989731699228, |
|
"rewards/rejected": -0.002419896889477968, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 4.946716615897932e-06, |
|
"logits/chosen": 0.11935708671808243, |
|
"logits/rejected": 0.18251103162765503, |
|
"logps/chosen": -354.6888732910156, |
|
"logps/rejected": -341.3345642089844, |
|
"loss": 0.3703, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": -0.0003122333437204361, |
|
"rewards/margins": 0.010582786984741688, |
|
"rewards/rejected": -0.010895021259784698, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 4.9142106826480114e-06, |
|
"logits/chosen": 0.06461165845394135, |
|
"logits/rejected": 0.18685249984264374, |
|
"logps/chosen": -362.90142822265625, |
|
"logps/rejected": -353.4388122558594, |
|
"loss": 0.3406, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -0.007556666620075703, |
|
"rewards/margins": 0.0255865640938282, |
|
"rewards/rejected": -0.03314322978258133, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 4.874162703221823e-06, |
|
"logits/chosen": 0.07998766005039215, |
|
"logits/rejected": 0.17009273171424866, |
|
"logps/chosen": -389.65362548828125, |
|
"logps/rejected": -363.02374267578125, |
|
"loss": 0.3605, |
|
"rewards/accuracies": 0.606249988079071, |
|
"rewards/chosen": -0.036845412105321884, |
|
"rewards/margins": 0.020548993721604347, |
|
"rewards/rejected": -0.05739440768957138, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 4.826697788369752e-06, |
|
"logits/chosen": 0.08132342249155045, |
|
"logits/rejected": 0.14768512547016144, |
|
"logps/chosen": -431.18109130859375, |
|
"logps/rejected": -458.8611755371094, |
|
"loss": 0.3284, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -0.06240760162472725, |
|
"rewards/margins": 0.06149229407310486, |
|
"rewards/rejected": -0.12389989197254181, |
|
"step": 130 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 4.7719642195082224e-06, |
|
"logits/chosen": 0.07962881028652191, |
|
"logits/rejected": 0.07184718549251556, |
|
"logps/chosen": -449.84393310546875, |
|
"logps/rejected": -531.15087890625, |
|
"loss": 0.2981, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -0.13113334774971008, |
|
"rewards/margins": 0.08747557550668716, |
|
"rewards/rejected": -0.21860890090465546, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 4.710132985485355e-06, |
|
"logits/chosen": 0.048229388892650604, |
|
"logits/rejected": 0.13614359498023987, |
|
"logps/chosen": -528.4464721679688, |
|
"logps/rejected": -578.7511596679688, |
|
"loss": 0.3243, |
|
"rewards/accuracies": 0.59375, |
|
"rewards/chosen": -0.18633897602558136, |
|
"rewards/margins": 0.07957583665847778, |
|
"rewards/rejected": -0.26591476798057556, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 4.641397248408122e-06, |
|
"logits/chosen": 0.11920128762722015, |
|
"logits/rejected": 0.15292052924633026, |
|
"logps/chosen": -552.9255981445312, |
|
"logps/rejected": -630.448486328125, |
|
"loss": 0.3063, |
|
"rewards/accuracies": 0.706250011920929, |
|
"rewards/chosen": -0.1733545958995819, |
|
"rewards/margins": 0.08863510191440582, |
|
"rewards/rejected": -0.2619897127151489, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 4.5659717401997655e-06, |
|
"logits/chosen": 0.06991209089756012, |
|
"logits/rejected": 0.12328378856182098, |
|
"logps/chosen": -470.7796325683594, |
|
"logps/rejected": -537.2589721679688, |
|
"loss": 0.3181, |
|
"rewards/accuracies": 0.6625000238418579, |
|
"rewards/chosen": -0.14576277136802673, |
|
"rewards/margins": 0.07790832221508026, |
|
"rewards/rejected": -0.2236711084842682, |
|
"step": 170 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 4.4840920917726425e-06, |
|
"logits/chosen": 0.128515362739563, |
|
"logits/rejected": 0.18245835602283478, |
|
"logps/chosen": -488.5677185058594, |
|
"logps/rejected": -617.6085205078125, |
|
"loss": 0.3053, |
|
"rewards/accuracies": 0.65625, |
|
"rewards/chosen": -0.15714891254901886, |
|
"rewards/margins": 0.1121206283569336, |
|
"rewards/rejected": -0.26926952600479126, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 4.396014096912182e-06, |
|
"logits/chosen": 0.06968175619840622, |
|
"logits/rejected": 0.1405131071805954, |
|
"logps/chosen": -457.48687744140625, |
|
"logps/rejected": -507.72491455078125, |
|
"loss": 0.3217, |
|
"rewards/accuracies": 0.65625, |
|
"rewards/chosen": -0.144833505153656, |
|
"rewards/margins": 0.07206179946660995, |
|
"rewards/rejected": -0.21689531207084656, |
|
"step": 190 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 4.302012913171584e-06, |
|
"logits/chosen": 0.11664710193872452, |
|
"logits/rejected": 0.1742321252822876, |
|
"logps/chosen": -498.94708251953125, |
|
"logps/rejected": -543.8130493164062, |
|
"loss": 0.3178, |
|
"rewards/accuracies": 0.606249988079071, |
|
"rewards/chosen": -0.1338798701763153, |
|
"rewards/margins": 0.07743647694587708, |
|
"rewards/rejected": -0.21131634712219238, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 4.202382202273702e-06, |
|
"logits/chosen": 0.09494765847921371, |
|
"logits/rejected": 0.1851937472820282, |
|
"logps/chosen": -520.176025390625, |
|
"logps/rejected": -567.4844970703125, |
|
"loss": 0.3061, |
|
"rewards/accuracies": 0.668749988079071, |
|
"rewards/chosen": -0.14064568281173706, |
|
"rewards/margins": 0.08944287896156311, |
|
"rewards/rejected": -0.23008854687213898, |
|
"step": 210 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 4.097433212705492e-06, |
|
"logits/chosen": 0.12784181535243988, |
|
"logits/rejected": 0.12376650422811508, |
|
"logps/chosen": -478.936767578125, |
|
"logps/rejected": -583.5801391601562, |
|
"loss": 0.3215, |
|
"rewards/accuracies": 0.6187499761581421, |
|
"rewards/chosen": -0.16416385769844055, |
|
"rewards/margins": 0.08470286428928375, |
|
"rewards/rejected": -0.2488667219877243, |
|
"step": 220 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 3.987493807371033e-06, |
|
"logits/chosen": 0.14440543949604034, |
|
"logits/rejected": 0.19996806979179382, |
|
"logps/chosen": -516.4754028320312, |
|
"logps/rejected": -626.3532104492188, |
|
"loss": 0.2979, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -0.1449437290430069, |
|
"rewards/margins": 0.12377123534679413, |
|
"rewards/rejected": -0.268714964389801, |
|
"step": 230 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 3.872907439340758e-06, |
|
"logits/chosen": 0.17143338918685913, |
|
"logits/rejected": 0.19446048140525818, |
|
"logps/chosen": -530.1805419921875, |
|
"logps/rejected": -642.0897827148438, |
|
"loss": 0.3007, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -0.1711062490940094, |
|
"rewards/margins": 0.11968924105167389, |
|
"rewards/rejected": -0.2907955050468445, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 3.75403207889666e-06, |
|
"logits/chosen": 0.13282717764377594, |
|
"logits/rejected": 0.22809620201587677, |
|
"logps/chosen": -522.2652587890625, |
|
"logps/rejected": -587.6581420898438, |
|
"loss": 0.3026, |
|
"rewards/accuracies": 0.668749988079071, |
|
"rewards/chosen": -0.16960139572620392, |
|
"rewards/margins": 0.09930792450904846, |
|
"rewards/rejected": -0.2689093053340912, |
|
"step": 250 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 3.631239095225417e-06, |
|
"logits/chosen": 0.17684991657733917, |
|
"logits/rejected": 0.17547938227653503, |
|
"logps/chosen": -496.5772399902344, |
|
"logps/rejected": -577.0806884765625, |
|
"loss": 0.3156, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -0.1634175181388855, |
|
"rewards/margins": 0.09483225643634796, |
|
"rewards/rejected": -0.25824978947639465, |
|
"step": 260 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 3.5049120962530608e-06, |
|
"logits/chosen": 0.09195151180028915, |
|
"logits/rejected": 0.21280470490455627, |
|
"logps/chosen": -544.9461669921875, |
|
"logps/rejected": -664.9347534179688, |
|
"loss": 0.2795, |
|
"rewards/accuracies": 0.6812499761581421, |
|
"rewards/chosen": -0.18792234361171722, |
|
"rewards/margins": 0.1312183439731598, |
|
"rewards/rejected": -0.3191407024860382, |
|
"step": 270 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 3.3754457302455464e-06, |
|
"logits/chosen": 0.14985263347625732, |
|
"logits/rejected": 0.22761961817741394, |
|
"logps/chosen": -607.9620361328125, |
|
"logps/rejected": -663.138427734375, |
|
"loss": 0.3365, |
|
"rewards/accuracies": 0.668749988079071, |
|
"rewards/chosen": -0.24214263260364532, |
|
"rewards/margins": 0.07931643724441528, |
|
"rewards/rejected": -0.3214590847492218, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 3.2432444529190714e-06, |
|
"logits/chosen": 0.15639743208885193, |
|
"logits/rejected": 0.23564644157886505, |
|
"logps/chosen": -586.5012817382812, |
|
"logps/rejected": -683.345947265625, |
|
"loss": 0.2713, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -0.21934108436107635, |
|
"rewards/margins": 0.14433029294013977, |
|
"rewards/rejected": -0.36367136240005493, |
|
"step": 290 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 3.1087212639117057e-06, |
|
"logits/chosen": 0.1709739863872528, |
|
"logits/rejected": 0.2245495617389679, |
|
"logps/chosen": -541.6049194335938, |
|
"logps/rejected": -656.5075073242188, |
|
"loss": 0.3202, |
|
"rewards/accuracies": 0.65625, |
|
"rewards/chosen": -0.21079179644584656, |
|
"rewards/margins": 0.11757495254278183, |
|
"rewards/rejected": -0.3283667266368866, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 2.9722964165636263e-06, |
|
"logits/chosen": 0.14368459582328796, |
|
"logits/rejected": 0.2023853063583374, |
|
"logps/chosen": -536.142822265625, |
|
"logps/rejected": -626.310546875, |
|
"loss": 0.3197, |
|
"rewards/accuracies": 0.6312500238418579, |
|
"rewards/chosen": -0.1972382515668869, |
|
"rewards/margins": 0.10271339118480682, |
|
"rewards/rejected": -0.2999516427516937, |
|
"step": 310 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 2.8343961050366275e-06, |
|
"logits/chosen": 0.10999581962823868, |
|
"logits/rejected": 0.19454975426197052, |
|
"logps/chosen": -585.6015625, |
|
"logps/rejected": -633.619140625, |
|
"loss": 0.3227, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": -0.18746989965438843, |
|
"rewards/margins": 0.09407122433185577, |
|
"rewards/rejected": -0.2815411686897278, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 2.695451132874385e-06, |
|
"logits/chosen": 0.12884345650672913, |
|
"logits/rejected": 0.21605506539344788, |
|
"logps/chosen": -508.71331787109375, |
|
"logps/rejected": -659.6421508789062, |
|
"loss": 0.2864, |
|
"rewards/accuracies": 0.65625, |
|
"rewards/chosen": -0.17948082089424133, |
|
"rewards/margins": 0.12334544956684113, |
|
"rewards/rejected": -0.30282628536224365, |
|
"step": 330 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 2.5558955671628964e-06, |
|
"logits/chosen": 0.1270858496427536, |
|
"logits/rejected": 0.17638283967971802, |
|
"logps/chosen": -505.85235595703125, |
|
"logps/rejected": -656.5067138671875, |
|
"loss": 0.2753, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -0.1641542613506317, |
|
"rewards/margins": 0.12927937507629395, |
|
"rewards/rejected": -0.29343363642692566, |
|
"step": 340 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 2.4161653824955654e-06, |
|
"logits/chosen": 0.15247094631195068, |
|
"logits/rejected": 0.14038711786270142, |
|
"logps/chosen": -518.3817138671875, |
|
"logps/rejected": -598.987548828125, |
|
"loss": 0.2873, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -0.181033656001091, |
|
"rewards/margins": 0.09822895377874374, |
|
"rewards/rejected": -0.27926260232925415, |
|
"step": 350 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 2.2766970989791697e-06, |
|
"logits/chosen": 0.14109697937965393, |
|
"logits/rejected": 0.25879988074302673, |
|
"logps/chosen": -527.6212158203125, |
|
"logps/rejected": -657.6229248046875, |
|
"loss": 0.2737, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -0.17991268634796143, |
|
"rewards/margins": 0.14120043814182281, |
|
"rewards/rejected": -0.32111310958862305, |
|
"step": 360 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 2.1379264185356545e-06, |
|
"logits/chosen": 0.18675974011421204, |
|
"logits/rejected": 0.2627638578414917, |
|
"logps/chosen": -579.1731567382812, |
|
"logps/rejected": -669.7673950195312, |
|
"loss": 0.2839, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -0.20380540192127228, |
|
"rewards/margins": 0.12797793745994568, |
|
"rewards/rejected": -0.33178335428237915, |
|
"step": 370 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 2.000286863759934e-06, |
|
"logits/chosen": 0.1522998809814453, |
|
"logits/rejected": 0.20338353514671326, |
|
"logps/chosen": -511.1458435058594, |
|
"logps/rejected": -621.6883544921875, |
|
"loss": 0.3078, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -0.1741541475057602, |
|
"rewards/margins": 0.13219627737998962, |
|
"rewards/rejected": -0.306350439786911, |
|
"step": 380 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 1.8642084235859764e-06, |
|
"logits/chosen": 0.16782529652118683, |
|
"logits/rejected": 0.25166839361190796, |
|
"logps/chosen": -559.2919921875, |
|
"logps/rejected": -642.5906372070312, |
|
"loss": 0.3013, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -0.17873121798038483, |
|
"rewards/margins": 0.09633437544107437, |
|
"rewards/rejected": -0.2750656008720398, |
|
"step": 390 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 1.7301162099921013e-06, |
|
"logits/chosen": 0.18475747108459473, |
|
"logits/rejected": 0.1979983150959015, |
|
"logps/chosen": -512.885009765625, |
|
"logps/rejected": -597.8982543945312, |
|
"loss": 0.3458, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -0.20149023830890656, |
|
"rewards/margins": 0.0695062130689621, |
|
"rewards/rejected": -0.2709963917732239, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 1.5984291299420117e-06, |
|
"logits/chosen": 0.13157054781913757, |
|
"logits/rejected": 0.21159140765666962, |
|
"logps/chosen": -526.1318359375, |
|
"logps/rejected": -639.5899658203125, |
|
"loss": 0.2671, |
|
"rewards/accuracies": 0.6875, |
|
"rewards/chosen": -0.1575133055448532, |
|
"rewards/margins": 0.13058245182037354, |
|
"rewards/rejected": -0.28809577226638794, |
|
"step": 410 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 1.4695585767104092e-06, |
|
"logits/chosen": 0.14387401938438416, |
|
"logits/rejected": 0.23105745017528534, |
|
"logps/chosen": -597.7404174804688, |
|
"logps/rejected": -684.18359375, |
|
"loss": 0.2713, |
|
"rewards/accuracies": 0.65625, |
|
"rewards/chosen": -0.18249225616455078, |
|
"rewards/margins": 0.12386848032474518, |
|
"rewards/rejected": -0.30636072158813477, |
|
"step": 420 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 1.3439071446815452e-06, |
|
"logits/chosen": 0.11251509189605713, |
|
"logits/rejected": 0.17909184098243713, |
|
"logps/chosen": -518.1268920898438, |
|
"logps/rejected": -573.087890625, |
|
"loss": 0.3273, |
|
"rewards/accuracies": 0.59375, |
|
"rewards/chosen": -0.1990637481212616, |
|
"rewards/margins": 0.07952968031167984, |
|
"rewards/rejected": -0.2785934507846832, |
|
"step": 430 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 1.2218673716356919e-06, |
|
"logits/chosen": 0.11003521829843521, |
|
"logits/rejected": 0.21477892994880676, |
|
"logps/chosen": -500.889404296875, |
|
"logps/rejected": -645.4175415039062, |
|
"loss": 0.285, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -0.1877116858959198, |
|
"rewards/margins": 0.13334953784942627, |
|
"rewards/rejected": -0.3210611939430237, |
|
"step": 440 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 1.103820512452661e-06, |
|
"logits/chosen": 0.1255572885274887, |
|
"logits/rejected": 0.22474932670593262, |
|
"logps/chosen": -600.7024536132812, |
|
"logps/rejected": -698.1460571289062, |
|
"loss": 0.2723, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -0.19767552614212036, |
|
"rewards/margins": 0.14586390554904938, |
|
"rewards/rejected": -0.34353941679000854, |
|
"step": 450 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 9.901353480633468e-07, |
|
"logits/chosen": 0.13007649779319763, |
|
"logits/rejected": 0.18805541098117828, |
|
"logps/chosen": -527.2013549804688, |
|
"logps/rejected": -628.3349609375, |
|
"loss": 0.3286, |
|
"rewards/accuracies": 0.606249988079071, |
|
"rewards/chosen": -0.22881360352039337, |
|
"rewards/margins": 0.09889401495456696, |
|
"rewards/rejected": -0.32770758867263794, |
|
"step": 460 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 8.811670333701544e-07, |
|
"logits/chosen": 0.10513252019882202, |
|
"logits/rejected": 0.1565053015947342, |
|
"logps/chosen": -563.048828125, |
|
"logps/rejected": -633.2500610351562, |
|
"loss": 0.2992, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": -0.20568466186523438, |
|
"rewards/margins": 0.10016629844903946, |
|
"rewards/rejected": -0.3058509826660156, |
|
"step": 470 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 7.772559877354341e-07, |
|
"logits/chosen": 0.1835373491048813, |
|
"logits/rejected": 0.19298210740089417, |
|
"logps/chosen": -548.7342529296875, |
|
"logps/rejected": -617.98681640625, |
|
"loss": 0.3108, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": -0.20563273131847382, |
|
"rewards/margins": 0.08818547427654266, |
|
"rewards/rejected": -0.2938181757926941, |
|
"step": 480 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 6.787268315040604e-07, |
|
"logits/chosen": 0.11183663457632065, |
|
"logits/rejected": 0.1670197993516922, |
|
"logps/chosen": -581.4060668945312, |
|
"logps/rejected": -655.2175903320312, |
|
"loss": 0.323, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -0.21741382777690887, |
|
"rewards/margins": 0.0863969624042511, |
|
"rewards/rejected": -0.3038107752799988, |
|
"step": 490 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 5.858873718824829e-07, |
|
"logits/chosen": 0.15304869413375854, |
|
"logits/rejected": 0.18997374176979065, |
|
"logps/chosen": -525.7789306640625, |
|
"logps/rejected": -628.69287109375, |
|
"loss": 0.2961, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": -0.188993901014328, |
|
"rewards/margins": 0.11667201668024063, |
|
"rewards/rejected": -0.30566591024398804, |
|
"step": 500 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 4.990276413423817e-07, |
|
"logits/chosen": 0.1436108499765396, |
|
"logits/rejected": 0.22793538868427277, |
|
"logps/chosen": -528.6312866210938, |
|
"logps/rejected": -641.5194091796875, |
|
"loss": 0.292, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.18891510367393494, |
|
"rewards/margins": 0.11953170597553253, |
|
"rewards/rejected": -0.30844682455062866, |
|
"step": 510 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 4.184189915529796e-07, |
|
"logits/chosen": 0.08341806381940842, |
|
"logits/rejected": 0.24343731999397278, |
|
"logps/chosen": -505.2992248535156, |
|
"logps/rejected": -625.7940673828125, |
|
"loss": 0.3127, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -0.18587087094783783, |
|
"rewards/margins": 0.10922978073358536, |
|
"rewards/rejected": -0.2951006591320038, |
|
"step": 520 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 3.4431324567258176e-07, |
|
"logits/chosen": 0.1076665073633194, |
|
"logits/rejected": 0.16546323895454407, |
|
"logps/chosen": -564.4067993164062, |
|
"logps/rejected": -617.0671997070312, |
|
"loss": 0.3086, |
|
"rewards/accuracies": 0.643750011920929, |
|
"rewards/chosen": -0.19417104125022888, |
|
"rewards/margins": 0.11286197602748871, |
|
"rewards/rejected": -0.3070330023765564, |
|
"step": 530 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 2.769419116476052e-07, |
|
"logits/chosen": 0.14298070967197418, |
|
"logits/rejected": 0.17989978194236755, |
|
"logps/chosen": -521.3731689453125, |
|
"logps/rejected": -603.6255493164062, |
|
"loss": 0.315, |
|
"rewards/accuracies": 0.612500011920929, |
|
"rewards/chosen": -0.19311046600341797, |
|
"rewards/margins": 0.09155095368623734, |
|
"rewards/rejected": -0.2846614718437195, |
|
"step": 540 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 2.1651545897676512e-07, |
|
"logits/chosen": 0.14457222819328308, |
|
"logits/rejected": 0.1904611885547638, |
|
"logps/chosen": -503.63385009765625, |
|
"logps/rejected": -589.3573608398438, |
|
"loss": 0.3095, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": -0.1882583498954773, |
|
"rewards/margins": 0.0975576862692833, |
|
"rewards/rejected": -0.2858160138130188, |
|
"step": 550 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 1.6322266119983222e-07, |
|
"logits/chosen": 0.1182747334241867, |
|
"logits/rejected": 0.21196098625659943, |
|
"logps/chosen": -558.52099609375, |
|
"logps/rejected": -705.6689453125, |
|
"loss": 0.2741, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -0.2015405148267746, |
|
"rewards/margins": 0.12743064761161804, |
|
"rewards/rejected": -0.32897114753723145, |
|
"step": 560 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 1.1723000616502167e-07, |
|
"logits/chosen": 0.13327908515930176, |
|
"logits/rejected": 0.2085684984922409, |
|
"logps/chosen": -520.0775146484375, |
|
"logps/rejected": -619.95703125, |
|
"loss": 0.2912, |
|
"rewards/accuracies": 0.668749988079071, |
|
"rewards/chosen": -0.18114188313484192, |
|
"rewards/margins": 0.1153794676065445, |
|
"rewards/rejected": -0.29652130603790283, |
|
"step": 570 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 7.868117591737585e-08, |
|
"logits/chosen": 0.11308556795120239, |
|
"logits/rejected": 0.16310641169548035, |
|
"logps/chosen": -502.0087890625, |
|
"logps/rejected": -638.6658935546875, |
|
"loss": 0.2833, |
|
"rewards/accuracies": 0.706250011920929, |
|
"rewards/chosen": -0.1852196305990219, |
|
"rewards/margins": 0.13148216903209686, |
|
"rewards/rejected": -0.3167017996311188, |
|
"step": 580 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 4.769659783295383e-08, |
|
"logits/chosen": 0.1447245329618454, |
|
"logits/rejected": 0.17959550023078918, |
|
"logps/chosen": -533.7676391601562, |
|
"logps/rejected": -665.7962036132812, |
|
"loss": 0.275, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -0.18203890323638916, |
|
"rewards/margins": 0.13183028995990753, |
|
"rewards/rejected": -0.3138691782951355, |
|
"step": 590 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.4373068401120358e-08, |
|
"logits/chosen": 0.1246277466416359, |
|
"logits/rejected": 0.19313006103038788, |
|
"logps/chosen": -532.855224609375, |
|
"logps/rejected": -646.3125, |
|
"loss": 0.3061, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": -0.18818099796772003, |
|
"rewards/margins": 0.12483028322458267, |
|
"rewards/rejected": -0.3130112886428833, |
|
"step": 600 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 8.78345083022425e-09, |
|
"logits/chosen": 0.07625824213027954, |
|
"logits/rejected": 0.19986259937286377, |
|
"logps/chosen": -546.0481567382812, |
|
"logps/rejected": -647.2523193359375, |
|
"loss": 0.2853, |
|
"rewards/accuracies": 0.6875, |
|
"rewards/chosen": -0.207177072763443, |
|
"rewards/margins": 0.1102672815322876, |
|
"rewards/rejected": -0.3174443542957306, |
|
"step": 610 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 9.764474213677654e-10, |
|
"logits/chosen": 0.08471451699733734, |
|
"logits/rejected": 0.1842333823442459, |
|
"logps/chosen": -539.9765014648438, |
|
"logps/rejected": -655.6905517578125, |
|
"loss": 0.2766, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -0.18608103692531586, |
|
"rewards/margins": 0.13810980319976807, |
|
"rewards/rejected": -0.3241908550262451, |
|
"step": 620 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"step": 625, |
|
"total_flos": 0.0, |
|
"train_loss": 0.31405294857025146, |
|
"train_runtime": 5821.3149, |
|
"train_samples_per_second": 1.718, |
|
"train_steps_per_second": 0.107 |
|
} |
|
], |
|
"logging_steps": 10, |
|
"max_steps": 625, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 100, |
|
"total_flos": 0.0, |
|
"train_batch_size": 4, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|