|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 0.9997382884061764, |
|
"eval_steps": 100, |
|
"global_step": 1910, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0005234231876472127, |
|
"grad_norm": 0.5695298687302295, |
|
"learning_rate": 2.617801047120419e-08, |
|
"logits/chosen": -0.4997953176498413, |
|
"logits/rejected": -0.5751151442527771, |
|
"logps/chosen": -395.12640380859375, |
|
"logps/rejected": -316.8270568847656, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.0, |
|
"rewards/chosen": 0.0, |
|
"rewards/margins": 0.0, |
|
"rewards/rejected": 0.0, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.005234231876472127, |
|
"grad_norm": 0.5536662200797415, |
|
"learning_rate": 2.617801047120419e-07, |
|
"logits/chosen": -0.5801360011100769, |
|
"logits/rejected": -0.6067044138908386, |
|
"logps/chosen": -304.32293701171875, |
|
"logps/rejected": -244.29046630859375, |
|
"loss": 0.6929, |
|
"rewards/accuracies": 0.4861111044883728, |
|
"rewards/chosen": -0.0005742510547861457, |
|
"rewards/margins": 0.00042081804713234305, |
|
"rewards/rejected": -0.0009950690437108278, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.010468463752944255, |
|
"grad_norm": 0.4593035639265128, |
|
"learning_rate": 5.235602094240838e-07, |
|
"logits/chosen": -0.6146650314331055, |
|
"logits/rejected": -0.615092396736145, |
|
"logps/chosen": -267.64739990234375, |
|
"logps/rejected": -261.7188415527344, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.4749999940395355, |
|
"rewards/chosen": 0.0001226421882165596, |
|
"rewards/margins": -0.00022562053345609456, |
|
"rewards/rejected": 0.00034826272167265415, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.015702695629416383, |
|
"grad_norm": 0.5145955390417808, |
|
"learning_rate": 7.853403141361258e-07, |
|
"logits/chosen": -0.6289754509925842, |
|
"logits/rejected": -0.6177533268928528, |
|
"logps/chosen": -280.3735046386719, |
|
"logps/rejected": -242.95175170898438, |
|
"loss": 0.693, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": 0.0002500644768588245, |
|
"rewards/margins": 0.0001505579421063885, |
|
"rewards/rejected": 9.950650564860553e-05, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.02093692750588851, |
|
"grad_norm": 0.5230612257382828, |
|
"learning_rate": 1.0471204188481676e-06, |
|
"logits/chosen": -0.5945444107055664, |
|
"logits/rejected": -0.6121580600738525, |
|
"logps/chosen": -267.5122375488281, |
|
"logps/rejected": -268.740234375, |
|
"loss": 0.6926, |
|
"rewards/accuracies": 0.5562499761581421, |
|
"rewards/chosen": 0.0005460727261379361, |
|
"rewards/margins": 0.0009088722290471196, |
|
"rewards/rejected": -0.0003627995611168444, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.02617115938236064, |
|
"grad_norm": 0.5459284518349332, |
|
"learning_rate": 1.3089005235602096e-06, |
|
"logits/chosen": -0.6024752259254456, |
|
"logits/rejected": -0.6329907774925232, |
|
"logps/chosen": -285.15838623046875, |
|
"logps/rejected": -254.0716552734375, |
|
"loss": 0.693, |
|
"rewards/accuracies": 0.574999988079071, |
|
"rewards/chosen": 0.0005030709435231984, |
|
"rewards/margins": 0.000754082459025085, |
|
"rewards/rejected": -0.0002510116610210389, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.031405391258832765, |
|
"grad_norm": 0.47503175617580706, |
|
"learning_rate": 1.5706806282722515e-06, |
|
"logits/chosen": -0.6266981363296509, |
|
"logits/rejected": -0.6208000183105469, |
|
"logps/chosen": -318.3113098144531, |
|
"logps/rejected": -271.3620300292969, |
|
"loss": 0.6917, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": 0.0019611348398029804, |
|
"rewards/margins": 0.002480647061020136, |
|
"rewards/rejected": -0.0005195126286707819, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.036639623135304895, |
|
"grad_norm": 0.5387833374092045, |
|
"learning_rate": 1.8324607329842933e-06, |
|
"logits/chosen": -0.5752447843551636, |
|
"logits/rejected": -0.6073333024978638, |
|
"logps/chosen": -274.0575866699219, |
|
"logps/rejected": -241.2310028076172, |
|
"loss": 0.6912, |
|
"rewards/accuracies": 0.6875, |
|
"rewards/chosen": 0.0042357188649475574, |
|
"rewards/margins": 0.0048388526774942875, |
|
"rewards/rejected": -0.0006031342782080173, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.04187385501177702, |
|
"grad_norm": 0.573484197995026, |
|
"learning_rate": 2.094240837696335e-06, |
|
"logits/chosen": -0.5686159133911133, |
|
"logits/rejected": -0.5909486413002014, |
|
"logps/chosen": -298.2327880859375, |
|
"logps/rejected": -275.91986083984375, |
|
"loss": 0.6902, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": 0.007856507785618305, |
|
"rewards/margins": 0.0063124834559857845, |
|
"rewards/rejected": 0.0015440242132171988, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.04710808688824915, |
|
"grad_norm": 0.5157342119342403, |
|
"learning_rate": 2.356020942408377e-06, |
|
"logits/chosen": -0.6013309955596924, |
|
"logits/rejected": -0.6156548857688904, |
|
"logps/chosen": -256.23602294921875, |
|
"logps/rejected": -235.50259399414062, |
|
"loss": 0.6896, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": 0.010481725446879864, |
|
"rewards/margins": 0.005771929398179054, |
|
"rewards/rejected": 0.0047097960487008095, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.05234231876472128, |
|
"grad_norm": 0.5281481398431945, |
|
"learning_rate": 2.617801047120419e-06, |
|
"logits/chosen": -0.5954197645187378, |
|
"logits/rejected": -0.6362258195877075, |
|
"logps/chosen": -246.4618682861328, |
|
"logps/rejected": -205.40481567382812, |
|
"loss": 0.6865, |
|
"rewards/accuracies": 0.6812499761581421, |
|
"rewards/chosen": 0.01956891641020775, |
|
"rewards/margins": 0.013755053281784058, |
|
"rewards/rejected": 0.005813863128423691, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.05234231876472128, |
|
"eval_logits/chosen": -0.5952818393707275, |
|
"eval_logits/rejected": -0.6047875285148621, |
|
"eval_logps/chosen": -274.37066650390625, |
|
"eval_logps/rejected": -252.90138244628906, |
|
"eval_loss": 0.685746967792511, |
|
"eval_rewards/accuracies": 0.6809999942779541, |
|
"eval_rewards/chosen": 0.02022642455995083, |
|
"eval_rewards/margins": 0.016626423224806786, |
|
"eval_rewards/rejected": 0.003600001335144043, |
|
"eval_runtime": 492.1641, |
|
"eval_samples_per_second": 4.064, |
|
"eval_steps_per_second": 0.254, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.05757655064119341, |
|
"grad_norm": 0.5272794123234011, |
|
"learning_rate": 2.8795811518324613e-06, |
|
"logits/chosen": -0.5827732682228088, |
|
"logits/rejected": -0.6242547035217285, |
|
"logps/chosen": -261.44049072265625, |
|
"logps/rejected": -211.53970336914062, |
|
"loss": 0.6829, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": 0.024311328306794167, |
|
"rewards/margins": 0.0224609337747097, |
|
"rewards/rejected": 0.0018503913888707757, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 0.06281078251766553, |
|
"grad_norm": 0.5281893762396359, |
|
"learning_rate": 3.141361256544503e-06, |
|
"logits/chosen": -0.5533467531204224, |
|
"logits/rejected": -0.5774653553962708, |
|
"logps/chosen": -323.4781799316406, |
|
"logps/rejected": -284.3753967285156, |
|
"loss": 0.6828, |
|
"rewards/accuracies": 0.65625, |
|
"rewards/chosen": 0.028400782495737076, |
|
"rewards/margins": 0.018748918548226357, |
|
"rewards/rejected": 0.009651863016188145, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 0.06804501439413765, |
|
"grad_norm": 0.5624037848656742, |
|
"learning_rate": 3.403141361256545e-06, |
|
"logits/chosen": -0.519309401512146, |
|
"logits/rejected": -0.538366436958313, |
|
"logps/chosen": -298.1244201660156, |
|
"logps/rejected": -273.02166748046875, |
|
"loss": 0.6764, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": 0.018718790262937546, |
|
"rewards/margins": 0.03766729682683945, |
|
"rewards/rejected": -0.0189485065639019, |
|
"step": 130 |
|
}, |
|
{ |
|
"epoch": 0.07327924627060979, |
|
"grad_norm": 0.6241582910982458, |
|
"learning_rate": 3.6649214659685865e-06, |
|
"logits/chosen": -0.6155737638473511, |
|
"logits/rejected": -0.615781307220459, |
|
"logps/chosen": -254.7561798095703, |
|
"logps/rejected": -252.4250030517578, |
|
"loss": 0.6661, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -0.0013563375687226653, |
|
"rewards/margins": 0.060362327843904495, |
|
"rewards/rejected": -0.0617186613380909, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 0.07851347814708191, |
|
"grad_norm": 0.7819020006868519, |
|
"learning_rate": 3.926701570680629e-06, |
|
"logits/chosen": -0.5579255819320679, |
|
"logits/rejected": -0.5662384033203125, |
|
"logps/chosen": -294.1602478027344, |
|
"logps/rejected": -283.9729309082031, |
|
"loss": 0.6542, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -0.06525762379169464, |
|
"rewards/margins": 0.0802302211523056, |
|
"rewards/rejected": -0.14548785984516144, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 0.08374771002355404, |
|
"grad_norm": 1.0082244738839943, |
|
"learning_rate": 4.18848167539267e-06, |
|
"logits/chosen": -0.6144393086433411, |
|
"logits/rejected": -0.6438087224960327, |
|
"logps/chosen": -293.11529541015625, |
|
"logps/rejected": -274.2389221191406, |
|
"loss": 0.6383, |
|
"rewards/accuracies": 0.6875, |
|
"rewards/chosen": -0.17805984616279602, |
|
"rewards/margins": 0.11895015090703964, |
|
"rewards/rejected": -0.29701000452041626, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 0.08898194190002617, |
|
"grad_norm": 1.0185195747317044, |
|
"learning_rate": 4.450261780104713e-06, |
|
"logits/chosen": -0.6413242220878601, |
|
"logits/rejected": -0.6541165709495544, |
|
"logps/chosen": -264.94549560546875, |
|
"logps/rejected": -280.5323486328125, |
|
"loss": 0.6307, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -0.06771933287382126, |
|
"rewards/margins": 0.1421503722667694, |
|
"rewards/rejected": -0.20986969769001007, |
|
"step": 170 |
|
}, |
|
{ |
|
"epoch": 0.0942161737764983, |
|
"grad_norm": 1.4632164660193285, |
|
"learning_rate": 4.712041884816754e-06, |
|
"logits/chosen": -0.8007810711860657, |
|
"logits/rejected": -0.8442818522453308, |
|
"logps/chosen": -335.73040771484375, |
|
"logps/rejected": -315.95654296875, |
|
"loss": 0.6239, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": -0.2423553764820099, |
|
"rewards/margins": 0.19543686509132385, |
|
"rewards/rejected": -0.43779221177101135, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 0.09945040565297043, |
|
"grad_norm": 1.4793919762168768, |
|
"learning_rate": 4.9738219895287965e-06, |
|
"logits/chosen": -0.8508358001708984, |
|
"logits/rejected": -0.9171808362007141, |
|
"logps/chosen": -319.34222412109375, |
|
"logps/rejected": -312.38458251953125, |
|
"loss": 0.5741, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -0.3020084500312805, |
|
"rewards/margins": 0.32925525307655334, |
|
"rewards/rejected": -0.6312636733055115, |
|
"step": 190 |
|
}, |
|
{ |
|
"epoch": 0.10468463752944256, |
|
"grad_norm": 1.3909979729045314, |
|
"learning_rate": 4.999661831436499e-06, |
|
"logits/chosen": -0.8342474699020386, |
|
"logits/rejected": -0.8582927584648132, |
|
"logps/chosen": -305.7598571777344, |
|
"logps/rejected": -329.6683044433594, |
|
"loss": 0.5773, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -0.3809957504272461, |
|
"rewards/margins": 0.3516261875629425, |
|
"rewards/rejected": -0.7326219081878662, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.10468463752944256, |
|
"eval_logits/chosen": -0.9088600277900696, |
|
"eval_logits/rejected": -0.9407602548599243, |
|
"eval_logps/chosen": -330.3779296875, |
|
"eval_logps/rejected": -347.161376953125, |
|
"eval_loss": 0.580152690410614, |
|
"eval_rewards/accuracies": 0.7080000042915344, |
|
"eval_rewards/chosen": -0.5398465991020203, |
|
"eval_rewards/margins": 0.3991530239582062, |
|
"eval_rewards/rejected": -0.9389996528625488, |
|
"eval_runtime": 493.0243, |
|
"eval_samples_per_second": 4.057, |
|
"eval_steps_per_second": 0.254, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.10991886940591468, |
|
"grad_norm": 2.2981570438136902, |
|
"learning_rate": 4.9984929711403395e-06, |
|
"logits/chosen": -0.8167268633842468, |
|
"logits/rejected": -0.8397541046142578, |
|
"logps/chosen": -311.28460693359375, |
|
"logps/rejected": -334.7868347167969, |
|
"loss": 0.5759, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": -0.6255691051483154, |
|
"rewards/margins": 0.4341684877872467, |
|
"rewards/rejected": -1.0597375631332397, |
|
"step": 210 |
|
}, |
|
{ |
|
"epoch": 0.11515310128238682, |
|
"grad_norm": 2.1371618628386155, |
|
"learning_rate": 4.996489634487865e-06, |
|
"logits/chosen": -0.8237009048461914, |
|
"logits/rejected": -0.8077519536018372, |
|
"logps/chosen": -394.8115539550781, |
|
"logps/rejected": -410.77276611328125, |
|
"loss": 0.5898, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -1.0387346744537354, |
|
"rewards/margins": 0.47027960419654846, |
|
"rewards/rejected": -1.509014368057251, |
|
"step": 220 |
|
}, |
|
{ |
|
"epoch": 0.12038733315885894, |
|
"grad_norm": 2.1931382967274287, |
|
"learning_rate": 4.9936524905772466e-06, |
|
"logits/chosen": -0.7595096826553345, |
|
"logits/rejected": -0.7923563718795776, |
|
"logps/chosen": -336.1356201171875, |
|
"logps/rejected": -397.56024169921875, |
|
"loss": 0.5126, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -0.9009682536125183, |
|
"rewards/margins": 0.6920944452285767, |
|
"rewards/rejected": -1.5930627584457397, |
|
"step": 230 |
|
}, |
|
{ |
|
"epoch": 0.12562156503533106, |
|
"grad_norm": 3.5772907228062776, |
|
"learning_rate": 4.9899824869915e-06, |
|
"logits/chosen": -0.8187972903251648, |
|
"logits/rejected": -0.8766587376594543, |
|
"logps/chosen": -422.3431701660156, |
|
"logps/rejected": -457.0511779785156, |
|
"loss": 0.5458, |
|
"rewards/accuracies": 0.706250011920929, |
|
"rewards/chosen": -1.592961311340332, |
|
"rewards/margins": 0.48186540603637695, |
|
"rewards/rejected": -2.074826717376709, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 0.13085579691180318, |
|
"grad_norm": 1.9360981239656954, |
|
"learning_rate": 4.985480849482012e-06, |
|
"logits/chosen": -0.8391903042793274, |
|
"logits/rejected": -0.8728139996528625, |
|
"logps/chosen": -411.3055725097656, |
|
"logps/rejected": -408.16082763671875, |
|
"loss": 0.5184, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -1.0500333309173584, |
|
"rewards/margins": 0.5413428544998169, |
|
"rewards/rejected": -1.5913760662078857, |
|
"step": 250 |
|
}, |
|
{ |
|
"epoch": 0.1360900287882753, |
|
"grad_norm": 3.8063942330720124, |
|
"learning_rate": 4.980149081559142e-06, |
|
"logits/chosen": -0.8690627813339233, |
|
"logits/rejected": -0.9096955060958862, |
|
"logps/chosen": -373.99871826171875, |
|
"logps/rejected": -421.1673889160156, |
|
"loss": 0.5362, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -0.9393401145935059, |
|
"rewards/margins": 0.7630335092544556, |
|
"rewards/rejected": -1.702373743057251, |
|
"step": 260 |
|
}, |
|
{ |
|
"epoch": 0.14132426066474746, |
|
"grad_norm": 2.3218168753541364, |
|
"learning_rate": 4.9739889639900655e-06, |
|
"logits/chosen": -0.8408964276313782, |
|
"logits/rejected": -0.8704169392585754, |
|
"logps/chosen": -416.67352294921875, |
|
"logps/rejected": -505.36279296875, |
|
"loss": 0.4785, |
|
"rewards/accuracies": 0.793749988079071, |
|
"rewards/chosen": -1.3642221689224243, |
|
"rewards/margins": 0.9632150530815125, |
|
"rewards/rejected": -2.327437162399292, |
|
"step": 270 |
|
}, |
|
{ |
|
"epoch": 0.14655849254121958, |
|
"grad_norm": 3.6911223439591407, |
|
"learning_rate": 4.967002554204009e-06, |
|
"logits/chosen": -0.9045387506484985, |
|
"logits/rejected": -0.9169967770576477, |
|
"logps/chosen": -359.976806640625, |
|
"logps/rejected": -423.2659606933594, |
|
"loss": 0.5433, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -0.8073030710220337, |
|
"rewards/margins": 0.6288052201271057, |
|
"rewards/rejected": -1.4361083507537842, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 0.1517927244176917, |
|
"grad_norm": 3.3423618640552095, |
|
"learning_rate": 4.959192185605089e-06, |
|
"logits/chosen": -0.9444735646247864, |
|
"logits/rejected": -0.9996434450149536, |
|
"logps/chosen": -437.74822998046875, |
|
"logps/rejected": -472.47003173828125, |
|
"loss": 0.5385, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -1.0655570030212402, |
|
"rewards/margins": 0.7227457761764526, |
|
"rewards/rejected": -1.7883027791976929, |
|
"step": 290 |
|
}, |
|
{ |
|
"epoch": 0.15702695629416383, |
|
"grad_norm": 1.7496473866481979, |
|
"learning_rate": 4.950560466792969e-06, |
|
"logits/chosen": -0.9565297961235046, |
|
"logits/rejected": -1.0268198251724243, |
|
"logps/chosen": -383.75982666015625, |
|
"logps/rejected": -431.46868896484375, |
|
"loss": 0.546, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -1.1211820840835571, |
|
"rewards/margins": 0.7908428907394409, |
|
"rewards/rejected": -1.9120250940322876, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.15702695629416383, |
|
"eval_logits/chosen": -1.0509591102600098, |
|
"eval_logits/rejected": -1.093695044517517, |
|
"eval_logps/chosen": -375.9070739746094, |
|
"eval_logps/rejected": -426.7812194824219, |
|
"eval_loss": 0.5337316393852234, |
|
"eval_rewards/accuracies": 0.7369999885559082, |
|
"eval_rewards/chosen": -0.9951376914978027, |
|
"eval_rewards/margins": 0.7400606274604797, |
|
"eval_rewards/rejected": -1.7351982593536377, |
|
"eval_runtime": 490.0828, |
|
"eval_samples_per_second": 4.081, |
|
"eval_steps_per_second": 0.255, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.16226118817063595, |
|
"grad_norm": 4.167924029619072, |
|
"learning_rate": 4.9411102806916185e-06, |
|
"logits/chosen": -0.9661039113998413, |
|
"logits/rejected": -1.0193841457366943, |
|
"logps/chosen": -386.1661071777344, |
|
"logps/rejected": -459.80059814453125, |
|
"loss": 0.4885, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -1.1400443315505981, |
|
"rewards/margins": 0.9499530792236328, |
|
"rewards/rejected": -2.0899975299835205, |
|
"step": 310 |
|
}, |
|
{ |
|
"epoch": 0.16749542004710807, |
|
"grad_norm": 3.7134888984526406, |
|
"learning_rate": 4.930844783586424e-06, |
|
"logits/chosen": -0.9965893030166626, |
|
"logits/rejected": -1.0922787189483643, |
|
"logps/chosen": -487.04620361328125, |
|
"logps/rejected": -534.34326171875, |
|
"loss": 0.5121, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -2.133643627166748, |
|
"rewards/margins": 0.8644089698791504, |
|
"rewards/rejected": -2.9980525970458984, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 0.17272965192358022, |
|
"grad_norm": 2.6372716992994625, |
|
"learning_rate": 4.919767404070033e-06, |
|
"logits/chosen": -1.044533371925354, |
|
"logits/rejected": -1.132899522781372, |
|
"logps/chosen": -446.17132568359375, |
|
"logps/rejected": -512.2128295898438, |
|
"loss": 0.5259, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -1.830833077430725, |
|
"rewards/margins": 0.8785037994384766, |
|
"rewards/rejected": -2.709336519241333, |
|
"step": 330 |
|
}, |
|
{ |
|
"epoch": 0.17796388380005235, |
|
"grad_norm": 2.585264829780915, |
|
"learning_rate": 4.907881841897216e-06, |
|
"logits/chosen": -1.0099565982818604, |
|
"logits/rejected": -1.0530710220336914, |
|
"logps/chosen": -442.65887451171875, |
|
"logps/rejected": -488.6505432128906, |
|
"loss": 0.4837, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -1.5058012008666992, |
|
"rewards/margins": 0.7143822908401489, |
|
"rewards/rejected": -2.2201833724975586, |
|
"step": 340 |
|
}, |
|
{ |
|
"epoch": 0.18319811567652447, |
|
"grad_norm": 3.3879317879760484, |
|
"learning_rate": 4.89519206674919e-06, |
|
"logits/chosen": -0.9927116632461548, |
|
"logits/rejected": -1.0346943140029907, |
|
"logps/chosen": -456.9622497558594, |
|
"logps/rejected": -577.3074951171875, |
|
"loss": 0.4784, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -2.0847067832946777, |
|
"rewards/margins": 1.002403974533081, |
|
"rewards/rejected": -3.0871105194091797, |
|
"step": 350 |
|
}, |
|
{ |
|
"epoch": 0.1884323475529966, |
|
"grad_norm": 3.57516881024075, |
|
"learning_rate": 4.881702316907769e-06, |
|
"logits/chosen": -1.0031338930130005, |
|
"logits/rejected": -1.0955650806427002, |
|
"logps/chosen": -503.44683837890625, |
|
"logps/rejected": -570.7683715820312, |
|
"loss": 0.4926, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -1.9374068975448608, |
|
"rewards/margins": 1.0033470392227173, |
|
"rewards/rejected": -2.940753936767578, |
|
"step": 360 |
|
}, |
|
{ |
|
"epoch": 0.19366657942946872, |
|
"grad_norm": 5.038599527660393, |
|
"learning_rate": 4.86741709783982e-06, |
|
"logits/chosen": -1.026948094367981, |
|
"logits/rejected": -1.0862653255462646, |
|
"logps/chosen": -428.8387145996094, |
|
"logps/rejected": -521.2025146484375, |
|
"loss": 0.4521, |
|
"rewards/accuracies": 0.8374999761581421, |
|
"rewards/chosen": -1.6744959354400635, |
|
"rewards/margins": 1.2723809480667114, |
|
"rewards/rejected": -2.9468765258789062, |
|
"step": 370 |
|
}, |
|
{ |
|
"epoch": 0.19890081130594087, |
|
"grad_norm": 2.580969896149703, |
|
"learning_rate": 4.852341180692471e-06, |
|
"logits/chosen": -1.0237780809402466, |
|
"logits/rejected": -1.0312225818634033, |
|
"logps/chosen": -460.7245178222656, |
|
"logps/rejected": -579.8516845703125, |
|
"loss": 0.4707, |
|
"rewards/accuracies": 0.8187500238418579, |
|
"rewards/chosen": -1.8517287969589233, |
|
"rewards/margins": 1.0035735368728638, |
|
"rewards/rejected": -2.855302333831787, |
|
"step": 380 |
|
}, |
|
{ |
|
"epoch": 0.204135043182413, |
|
"grad_norm": 4.049123524870898, |
|
"learning_rate": 4.836479600699579e-06, |
|
"logits/chosen": -1.021194577217102, |
|
"logits/rejected": -1.065612554550171, |
|
"logps/chosen": -474.7022399902344, |
|
"logps/rejected": -515.19384765625, |
|
"loss": 0.5062, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -1.6703779697418213, |
|
"rewards/margins": 0.9831420183181763, |
|
"rewards/rejected": -2.653519868850708, |
|
"step": 390 |
|
}, |
|
{ |
|
"epoch": 0.2093692750588851, |
|
"grad_norm": 2.669349458066594, |
|
"learning_rate": 4.819837655500014e-06, |
|
"logits/chosen": -1.055345892906189, |
|
"logits/rejected": -1.1153924465179443, |
|
"logps/chosen": -409.0503845214844, |
|
"logps/rejected": -475.8119201660156, |
|
"loss": 0.501, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -1.5296335220336914, |
|
"rewards/margins": 0.9046236872673035, |
|
"rewards/rejected": -2.4342570304870605, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 0.2093692750588851, |
|
"eval_logits/chosen": -1.0595479011535645, |
|
"eval_logits/rejected": -1.101104974746704, |
|
"eval_logps/chosen": -458.5479431152344, |
|
"eval_logps/rejected": -529.427734375, |
|
"eval_loss": 0.5120114088058472, |
|
"eval_rewards/accuracies": 0.753000020980835, |
|
"eval_rewards/chosen": -1.8215464353561401, |
|
"eval_rewards/margins": 0.9401166439056396, |
|
"eval_rewards/rejected": -2.7616631984710693, |
|
"eval_runtime": 490.6583, |
|
"eval_samples_per_second": 4.076, |
|
"eval_steps_per_second": 0.255, |
|
"step": 400 |
|
}, |
|
{ |
|
"epoch": 0.21460350693535724, |
|
"grad_norm": 2.693659265285776, |
|
"learning_rate": 4.802420903368286e-06, |
|
"logits/chosen": -1.017884612083435, |
|
"logits/rejected": -1.0450990200042725, |
|
"logps/chosen": -452.4844665527344, |
|
"logps/rejected": -545.2508544921875, |
|
"loss": 0.5075, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -1.978095293045044, |
|
"rewards/margins": 0.8133966326713562, |
|
"rewards/rejected": -2.791491746902466, |
|
"step": 410 |
|
}, |
|
{ |
|
"epoch": 0.21983773881182936, |
|
"grad_norm": 3.0138584618040816, |
|
"learning_rate": 4.784235161358124e-06, |
|
"logits/chosen": -1.0173218250274658, |
|
"logits/rejected": -1.0448085069656372, |
|
"logps/chosen": -483.49609375, |
|
"logps/rejected": -571.6468505859375, |
|
"loss": 0.5192, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -2.0148584842681885, |
|
"rewards/margins": 0.8309059143066406, |
|
"rewards/rejected": -2.84576416015625, |
|
"step": 420 |
|
}, |
|
{ |
|
"epoch": 0.22507197068830148, |
|
"grad_norm": 2.7069812702685327, |
|
"learning_rate": 4.765286503359632e-06, |
|
"logits/chosen": -0.953274130821228, |
|
"logits/rejected": -1.0237443447113037, |
|
"logps/chosen": -445.3104553222656, |
|
"logps/rejected": -500.704345703125, |
|
"loss": 0.5173, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -1.7284530401229858, |
|
"rewards/margins": 0.7751290798187256, |
|
"rewards/rejected": -2.503582239151001, |
|
"step": 430 |
|
}, |
|
{ |
|
"epoch": 0.23030620256477363, |
|
"grad_norm": 2.6842575235901256, |
|
"learning_rate": 4.745581258070654e-06, |
|
"logits/chosen": -0.9820082783699036, |
|
"logits/rejected": -1.0258147716522217, |
|
"logps/chosen": -424.865966796875, |
|
"logps/rejected": -499.13031005859375, |
|
"loss": 0.5075, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -1.5546586513519287, |
|
"rewards/margins": 0.8893228769302368, |
|
"rewards/rejected": -2.443981409072876, |
|
"step": 440 |
|
}, |
|
{ |
|
"epoch": 0.23554043444124576, |
|
"grad_norm": 2.8248158426262036, |
|
"learning_rate": 4.725126006883047e-06, |
|
"logits/chosen": -0.9185377955436707, |
|
"logits/rejected": -0.94728022813797, |
|
"logps/chosen": -403.70159912109375, |
|
"logps/rejected": -514.3756713867188, |
|
"loss": 0.5284, |
|
"rewards/accuracies": 0.6875, |
|
"rewards/chosen": -1.593689203262329, |
|
"rewards/margins": 0.8767625689506531, |
|
"rewards/rejected": -2.470451593399048, |
|
"step": 450 |
|
}, |
|
{ |
|
"epoch": 0.24077466631771788, |
|
"grad_norm": 3.479296042367863, |
|
"learning_rate": 4.70392758168454e-06, |
|
"logits/chosen": -0.8726640939712524, |
|
"logits/rejected": -0.9334267377853394, |
|
"logps/chosen": -439.721435546875, |
|
"logps/rejected": -487.37384033203125, |
|
"loss": 0.5126, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -1.5047967433929443, |
|
"rewards/margins": 0.8623396158218384, |
|
"rewards/rejected": -2.3671364784240723, |
|
"step": 460 |
|
}, |
|
{ |
|
"epoch": 0.24600889819419, |
|
"grad_norm": 4.319750283791727, |
|
"learning_rate": 4.68199306257695e-06, |
|
"logits/chosen": -0.9039748311042786, |
|
"logits/rejected": -0.9309199452400208, |
|
"logps/chosen": -492.0690002441406, |
|
"logps/rejected": -554.73388671875, |
|
"loss": 0.4953, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -1.8066781759262085, |
|
"rewards/margins": 0.9244368672370911, |
|
"rewards/rejected": -2.7311148643493652, |
|
"step": 470 |
|
}, |
|
{ |
|
"epoch": 0.2512431300706621, |
|
"grad_norm": 3.14164730805949, |
|
"learning_rate": 4.659329775511478e-06, |
|
"logits/chosen": -0.9303410649299622, |
|
"logits/rejected": -0.977057933807373, |
|
"logps/chosen": -399.0423889160156, |
|
"logps/rejected": -490.00543212890625, |
|
"loss": 0.4683, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -1.4180445671081543, |
|
"rewards/margins": 1.0890886783599854, |
|
"rewards/rejected": -2.5071330070495605, |
|
"step": 480 |
|
}, |
|
{ |
|
"epoch": 0.2564773619471343, |
|
"grad_norm": 2.3614868571221357, |
|
"learning_rate": 4.635945289841902e-06, |
|
"logits/chosen": -0.8816567659378052, |
|
"logits/rejected": -0.9269768595695496, |
|
"logps/chosen": -415.19219970703125, |
|
"logps/rejected": -476.89556884765625, |
|
"loss": 0.4735, |
|
"rewards/accuracies": 0.6875, |
|
"rewards/chosen": -1.3752299547195435, |
|
"rewards/margins": 0.9180902242660522, |
|
"rewards/rejected": -2.2933201789855957, |
|
"step": 490 |
|
}, |
|
{ |
|
"epoch": 0.26171159382360637, |
|
"grad_norm": 2.8583338952559925, |
|
"learning_rate": 4.611847415796476e-06, |
|
"logits/chosen": -0.8435959815979004, |
|
"logits/rejected": -0.9220407605171204, |
|
"logps/chosen": -441.06201171875, |
|
"logps/rejected": -533.26904296875, |
|
"loss": 0.4525, |
|
"rewards/accuracies": 0.862500011920929, |
|
"rewards/chosen": -1.673166275024414, |
|
"rewards/margins": 1.2388008832931519, |
|
"rewards/rejected": -2.9119672775268555, |
|
"step": 500 |
|
}, |
|
{ |
|
"epoch": 0.26171159382360637, |
|
"eval_logits/chosen": -0.9134161472320557, |
|
"eval_logits/rejected": -0.9429917931556702, |
|
"eval_logps/chosen": -474.96240234375, |
|
"eval_logps/rejected": -561.74462890625, |
|
"eval_loss": 0.5090299248695374, |
|
"eval_rewards/accuracies": 0.7509999871253967, |
|
"eval_rewards/chosen": -1.985690951347351, |
|
"eval_rewards/margins": 1.0991418361663818, |
|
"eval_rewards/rejected": -3.0848329067230225, |
|
"eval_runtime": 490.4743, |
|
"eval_samples_per_second": 4.078, |
|
"eval_steps_per_second": 0.255, |
|
"step": 500 |
|
}, |
|
{ |
|
"epoch": 0.2669458257000785, |
|
"grad_norm": 3.035220836230476, |
|
"learning_rate": 4.587044201869378e-06, |
|
"logits/chosen": -0.8761506080627441, |
|
"logits/rejected": -0.902166485786438, |
|
"logps/chosen": -425.97332763671875, |
|
"logps/rejected": -501.02789306640625, |
|
"loss": 0.497, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -1.6953567266464233, |
|
"rewards/margins": 0.9890697598457336, |
|
"rewards/rejected": -2.6844265460968018, |
|
"step": 510 |
|
}, |
|
{ |
|
"epoch": 0.2721800575765506, |
|
"grad_norm": 3.130477955866568, |
|
"learning_rate": 4.561543932132574e-06, |
|
"logits/chosen": -0.7917976379394531, |
|
"logits/rejected": -0.8613120317459106, |
|
"logps/chosen": -403.6101379394531, |
|
"logps/rejected": -495.6336975097656, |
|
"loss": 0.4638, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -1.1837232112884521, |
|
"rewards/margins": 1.1299959421157837, |
|
"rewards/rejected": -2.3137192726135254, |
|
"step": 520 |
|
}, |
|
{ |
|
"epoch": 0.27741428945302277, |
|
"grad_norm": 5.62051791025711, |
|
"learning_rate": 4.535355123469009e-06, |
|
"logits/chosen": -0.8685650825500488, |
|
"logits/rejected": -0.9064447283744812, |
|
"logps/chosen": -453.49505615234375, |
|
"logps/rejected": -571.8440551757812, |
|
"loss": 0.4962, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -1.7710943222045898, |
|
"rewards/margins": 1.2530162334442139, |
|
"rewards/rejected": -3.0241103172302246, |
|
"step": 530 |
|
}, |
|
{ |
|
"epoch": 0.2826485213294949, |
|
"grad_norm": 4.031206237193703, |
|
"learning_rate": 4.508486522728037e-06, |
|
"logits/chosen": -0.8691636919975281, |
|
"logits/rejected": -0.8954900503158569, |
|
"logps/chosen": -481.83001708984375, |
|
"logps/rejected": -557.0582275390625, |
|
"loss": 0.464, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -2.1070663928985596, |
|
"rewards/margins": 1.1235601902008057, |
|
"rewards/rejected": -3.2306265830993652, |
|
"step": 540 |
|
}, |
|
{ |
|
"epoch": 0.287882753205967, |
|
"grad_norm": 2.5570522477618023, |
|
"learning_rate": 4.480947103804044e-06, |
|
"logits/chosen": -0.8274758458137512, |
|
"logits/rejected": -0.8582462072372437, |
|
"logps/chosen": -516.5406494140625, |
|
"logps/rejected": -561.04736328125, |
|
"loss": 0.5267, |
|
"rewards/accuracies": 0.793749988079071, |
|
"rewards/chosen": -2.3619277477264404, |
|
"rewards/margins": 0.9269983172416687, |
|
"rewards/rejected": -3.288925886154175, |
|
"step": 550 |
|
}, |
|
{ |
|
"epoch": 0.29311698508243916, |
|
"grad_norm": 2.788640181418083, |
|
"learning_rate": 4.452746064639239e-06, |
|
"logits/chosen": -0.9235193133354187, |
|
"logits/rejected": -0.9567066431045532, |
|
"logps/chosen": -500.55810546875, |
|
"logps/rejected": -591.5838012695312, |
|
"loss": 0.4856, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -2.2232844829559326, |
|
"rewards/margins": 0.9942595362663269, |
|
"rewards/rejected": -3.217543840408325, |
|
"step": 560 |
|
}, |
|
{ |
|
"epoch": 0.29835121695891126, |
|
"grad_norm": 2.993526317828841, |
|
"learning_rate": 4.423892824151617e-06, |
|
"logits/chosen": -0.9176028966903687, |
|
"logits/rejected": -0.9547454118728638, |
|
"logps/chosen": -483.21490478515625, |
|
"logps/rejected": -539.8753051757812, |
|
"loss": 0.4752, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -1.7947994470596313, |
|
"rewards/margins": 0.9781554937362671, |
|
"rewards/rejected": -2.7729547023773193, |
|
"step": 570 |
|
}, |
|
{ |
|
"epoch": 0.3035854488353834, |
|
"grad_norm": 3.5431736202634885, |
|
"learning_rate": 4.3943970190891164e-06, |
|
"logits/chosen": -0.9206205606460571, |
|
"logits/rejected": -0.9628446698188782, |
|
"logps/chosen": -463.5152282714844, |
|
"logps/rejected": -528.37158203125, |
|
"loss": 0.4885, |
|
"rewards/accuracies": 0.793749988079071, |
|
"rewards/chosen": -1.6288913488388062, |
|
"rewards/margins": 1.1642273664474487, |
|
"rewards/rejected": -2.793118953704834, |
|
"step": 580 |
|
}, |
|
{ |
|
"epoch": 0.30881968071185556, |
|
"grad_norm": 3.365610293148701, |
|
"learning_rate": 4.364268500811025e-06, |
|
"logits/chosen": -0.8485568165779114, |
|
"logits/rejected": -0.9199141263961792, |
|
"logps/chosen": -483.6082458496094, |
|
"logps/rejected": -560.9791259765625, |
|
"loss": 0.5057, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -2.1659016609191895, |
|
"rewards/margins": 1.034793734550476, |
|
"rewards/rejected": -3.200695514678955, |
|
"step": 590 |
|
}, |
|
{ |
|
"epoch": 0.31405391258832765, |
|
"grad_norm": 4.5918534300247975, |
|
"learning_rate": 4.333517331997704e-06, |
|
"logits/chosen": -0.9025293588638306, |
|
"logits/rejected": -0.9563030004501343, |
|
"logps/chosen": -486.9297790527344, |
|
"logps/rejected": -560.46630859375, |
|
"loss": 0.508, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -2.339505434036255, |
|
"rewards/margins": 0.8578627705574036, |
|
"rewards/rejected": -3.1973681449890137, |
|
"step": 600 |
|
}, |
|
{ |
|
"epoch": 0.31405391258832765, |
|
"eval_logits/chosen": -0.96256422996521, |
|
"eval_logits/rejected": -0.9954620599746704, |
|
"eval_logps/chosen": -497.45501708984375, |
|
"eval_logps/rejected": -568.3763427734375, |
|
"eval_loss": 0.5005487203598022, |
|
"eval_rewards/accuracies": 0.7599999904632568, |
|
"eval_rewards/chosen": -2.210617780685425, |
|
"eval_rewards/margins": 0.940531849861145, |
|
"eval_rewards/rejected": -3.151149272918701, |
|
"eval_runtime": 491.9737, |
|
"eval_samples_per_second": 4.065, |
|
"eval_steps_per_second": 0.254, |
|
"step": 600 |
|
}, |
|
{ |
|
"epoch": 0.3192881444647998, |
|
"grad_norm": 2.901694328327758, |
|
"learning_rate": 4.302153783289737e-06, |
|
"logits/chosen": -0.9074804186820984, |
|
"logits/rejected": -0.9480104446411133, |
|
"logps/chosen": -492.01910400390625, |
|
"logps/rejected": -600.9209594726562, |
|
"loss": 0.469, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -2.1020569801330566, |
|
"rewards/margins": 1.1244044303894043, |
|
"rewards/rejected": -3.226461410522461, |
|
"step": 610 |
|
}, |
|
{ |
|
"epoch": 0.3245223763412719, |
|
"grad_norm": 3.94946155728758, |
|
"learning_rate": 4.270188329857613e-06, |
|
"logits/chosen": -0.8973082304000854, |
|
"logits/rejected": -0.9559001922607422, |
|
"logps/chosen": -465.6111755371094, |
|
"logps/rejected": -525.2369384765625, |
|
"loss": 0.4846, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -1.6850999593734741, |
|
"rewards/margins": 1.1260297298431396, |
|
"rewards/rejected": -2.811129570007324, |
|
"step": 620 |
|
}, |
|
{ |
|
"epoch": 0.32975660821774405, |
|
"grad_norm": 3.033290372148656, |
|
"learning_rate": 4.237631647903115e-06, |
|
"logits/chosen": -0.9191315770149231, |
|
"logits/rejected": -0.9655174016952515, |
|
"logps/chosen": -424.0669860839844, |
|
"logps/rejected": -516.409912109375, |
|
"loss": 0.5221, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -1.6103105545043945, |
|
"rewards/margins": 1.0200879573822021, |
|
"rewards/rejected": -2.630398988723755, |
|
"step": 630 |
|
}, |
|
{ |
|
"epoch": 0.33499084009421615, |
|
"grad_norm": 6.947248145776445, |
|
"learning_rate": 4.204494611093548e-06, |
|
"logits/chosen": -0.8765581846237183, |
|
"logits/rejected": -0.9510295987129211, |
|
"logps/chosen": -403.50701904296875, |
|
"logps/rejected": -494.7275390625, |
|
"loss": 0.5006, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -1.570704698562622, |
|
"rewards/margins": 1.1784632205963135, |
|
"rewards/rejected": -2.7491679191589355, |
|
"step": 640 |
|
}, |
|
{ |
|
"epoch": 0.3402250719706883, |
|
"grad_norm": 3.486686904718388, |
|
"learning_rate": 4.170788286930024e-06, |
|
"logits/chosen": -0.9069339632987976, |
|
"logits/rejected": -0.9199384450912476, |
|
"logps/chosen": -444.85009765625, |
|
"logps/rejected": -555.9014282226562, |
|
"loss": 0.5045, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -1.9438743591308594, |
|
"rewards/margins": 1.0122339725494385, |
|
"rewards/rejected": -2.956108570098877, |
|
"step": 650 |
|
}, |
|
{ |
|
"epoch": 0.34545930384716045, |
|
"grad_norm": 4.061094803349214, |
|
"learning_rate": 4.136523933051005e-06, |
|
"logits/chosen": -0.8677660822868347, |
|
"logits/rejected": -0.9190397262573242, |
|
"logps/chosen": -406.159912109375, |
|
"logps/rejected": -498.9130859375, |
|
"loss": 0.497, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -1.4709824323654175, |
|
"rewards/margins": 0.9458149671554565, |
|
"rewards/rejected": -2.416797399520874, |
|
"step": 660 |
|
}, |
|
{ |
|
"epoch": 0.35069353572363254, |
|
"grad_norm": 4.528134734617534, |
|
"learning_rate": 4.101712993472348e-06, |
|
"logits/chosen": -0.8407672643661499, |
|
"logits/rejected": -0.8770118951797485, |
|
"logps/chosen": -421.9403381347656, |
|
"logps/rejected": -511.84820556640625, |
|
"loss": 0.5147, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -1.5217100381851196, |
|
"rewards/margins": 1.0356905460357666, |
|
"rewards/rejected": -2.5574002265930176, |
|
"step": 670 |
|
}, |
|
{ |
|
"epoch": 0.3559277676001047, |
|
"grad_norm": 3.331708093137532, |
|
"learning_rate": 4.066367094765091e-06, |
|
"logits/chosen": -0.8362857103347778, |
|
"logits/rejected": -0.8756265640258789, |
|
"logps/chosen": -427.3866271972656, |
|
"logps/rejected": -472.51116943359375, |
|
"loss": 0.5107, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -1.6107534170150757, |
|
"rewards/margins": 0.761100709438324, |
|
"rewards/rejected": -2.371854066848755, |
|
"step": 680 |
|
}, |
|
{ |
|
"epoch": 0.3611619994765768, |
|
"grad_norm": 3.054351768159171, |
|
"learning_rate": 4.030498042172277e-06, |
|
"logits/chosen": -0.887158989906311, |
|
"logits/rejected": -0.9305670857429504, |
|
"logps/chosen": -436.98663330078125, |
|
"logps/rejected": -536.4581298828125, |
|
"loss": 0.4933, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -1.5907622575759888, |
|
"rewards/margins": 1.0380842685699463, |
|
"rewards/rejected": -2.6288466453552246, |
|
"step": 690 |
|
}, |
|
{ |
|
"epoch": 0.36639623135304894, |
|
"grad_norm": 3.9217623567649165, |
|
"learning_rate": 3.994117815666095e-06, |
|
"logits/chosen": -0.8911476135253906, |
|
"logits/rejected": -0.9339841604232788, |
|
"logps/chosen": -435.4269104003906, |
|
"logps/rejected": -515.2606201171875, |
|
"loss": 0.4852, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -1.6368818283081055, |
|
"rewards/margins": 0.932881236076355, |
|
"rewards/rejected": -2.56976318359375, |
|
"step": 700 |
|
}, |
|
{ |
|
"epoch": 0.36639623135304894, |
|
"eval_logits/chosen": -0.9476205110549927, |
|
"eval_logits/rejected": -0.9794169664382935, |
|
"eval_logps/chosen": -416.10260009765625, |
|
"eval_logps/rejected": -494.5317077636719, |
|
"eval_loss": 0.5027905106544495, |
|
"eval_rewards/accuracies": 0.7770000100135803, |
|
"eval_rewards/chosen": -1.3970927000045776, |
|
"eval_rewards/margins": 1.0156108140945435, |
|
"eval_rewards/rejected": -2.412703275680542, |
|
"eval_runtime": 489.1035, |
|
"eval_samples_per_second": 4.089, |
|
"eval_steps_per_second": 0.256, |
|
"step": 700 |
|
}, |
|
{ |
|
"epoch": 0.3716304632295211, |
|
"grad_norm": 3.6606573607441772, |
|
"learning_rate": 3.957238565946672e-06, |
|
"logits/chosen": -0.8809655904769897, |
|
"logits/rejected": -0.9375411868095398, |
|
"logps/chosen": -473.4790954589844, |
|
"logps/rejected": -525.9794311523438, |
|
"loss": 0.4515, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -1.573014736175537, |
|
"rewards/margins": 1.1010257005691528, |
|
"rewards/rejected": -2.6740405559539795, |
|
"step": 710 |
|
}, |
|
{ |
|
"epoch": 0.3768646951059932, |
|
"grad_norm": 4.5368251934335655, |
|
"learning_rate": 3.919872610383831e-06, |
|
"logits/chosen": -0.8901004791259766, |
|
"logits/rejected": -0.9005835652351379, |
|
"logps/chosen": -456.73626708984375, |
|
"logps/rejected": -578.81201171875, |
|
"loss": 0.4683, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -2.028618335723877, |
|
"rewards/margins": 1.2738052606582642, |
|
"rewards/rejected": -3.3024234771728516, |
|
"step": 720 |
|
}, |
|
{ |
|
"epoch": 0.38209892698246534, |
|
"grad_norm": 4.7885163624566704, |
|
"learning_rate": 3.882032428903195e-06, |
|
"logits/chosen": -0.8851186633110046, |
|
"logits/rejected": -0.8868842124938965, |
|
"logps/chosen": -499.48040771484375, |
|
"logps/rejected": -608.1116333007812, |
|
"loss": 0.5104, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -2.6055073738098145, |
|
"rewards/margins": 1.1688528060913086, |
|
"rewards/rejected": -3.774359941482544, |
|
"step": 730 |
|
}, |
|
{ |
|
"epoch": 0.38733315885893743, |
|
"grad_norm": 2.6857761500293766, |
|
"learning_rate": 3.84373065981799e-06, |
|
"logits/chosen": -0.8212822675704956, |
|
"logits/rejected": -0.851559042930603, |
|
"logps/chosen": -512.4810791015625, |
|
"logps/rejected": -562.7243041992188, |
|
"loss": 0.451, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -2.420043468475342, |
|
"rewards/margins": 1.00551438331604, |
|
"rewards/rejected": -3.425558090209961, |
|
"step": 740 |
|
}, |
|
{ |
|
"epoch": 0.3925673907354096, |
|
"grad_norm": 5.276994078087987, |
|
"learning_rate": 3.8049800956079552e-06, |
|
"logits/chosen": -0.8459002375602722, |
|
"logits/rejected": -0.8481931686401367, |
|
"logps/chosen": -452.31103515625, |
|
"logps/rejected": -549.0643310546875, |
|
"loss": 0.4316, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -1.8753325939178467, |
|
"rewards/margins": 1.1157124042510986, |
|
"rewards/rejected": -2.9910449981689453, |
|
"step": 750 |
|
}, |
|
{ |
|
"epoch": 0.39780162261188173, |
|
"grad_norm": 4.2662925093714525, |
|
"learning_rate": 3.765793678646753e-06, |
|
"logits/chosen": -0.8927903175354004, |
|
"logits/rejected": -0.9021077156066895, |
|
"logps/chosen": -400.74273681640625, |
|
"logps/rejected": -483.39813232421875, |
|
"loss": 0.538, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -1.5194071531295776, |
|
"rewards/margins": 1.0460177659988403, |
|
"rewards/rejected": -2.565425157546997, |
|
"step": 760 |
|
}, |
|
{ |
|
"epoch": 0.40303585448835383, |
|
"grad_norm": 2.642122221399109, |
|
"learning_rate": 3.726184496879323e-06, |
|
"logits/chosen": -0.8244895935058594, |
|
"logits/rejected": -0.8434764742851257, |
|
"logps/chosen": -467.0747985839844, |
|
"logps/rejected": -540.9425048828125, |
|
"loss": 0.5261, |
|
"rewards/accuracies": 0.668749988079071, |
|
"rewards/chosen": -1.963444471359253, |
|
"rewards/margins": 0.8455499410629272, |
|
"rewards/rejected": -2.8089945316314697, |
|
"step": 770 |
|
}, |
|
{ |
|
"epoch": 0.408270086364826, |
|
"grad_norm": 3.728031543819746, |
|
"learning_rate": 3.686165779450619e-06, |
|
"logits/chosen": -0.8528604507446289, |
|
"logits/rejected": -0.8891122937202454, |
|
"logps/chosen": -460.427734375, |
|
"logps/rejected": -544.4713134765625, |
|
"loss": 0.4439, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -1.879953145980835, |
|
"rewards/margins": 1.079024314880371, |
|
"rewards/rejected": -2.958977699279785, |
|
"step": 780 |
|
}, |
|
{ |
|
"epoch": 0.4135043182412981, |
|
"grad_norm": 2.85654604962964, |
|
"learning_rate": 3.645750892287178e-06, |
|
"logits/chosen": -0.8759455680847168, |
|
"logits/rejected": -0.8966878652572632, |
|
"logps/chosen": -451.11865234375, |
|
"logps/rejected": -574.4451293945312, |
|
"loss": 0.5208, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -1.9754890203475952, |
|
"rewards/margins": 1.0046672821044922, |
|
"rewards/rejected": -2.980156421661377, |
|
"step": 790 |
|
}, |
|
{ |
|
"epoch": 0.4187385501177702, |
|
"grad_norm": 2.7417872161100374, |
|
"learning_rate": 3.604953333633009e-06, |
|
"logits/chosen": -0.8261939883232117, |
|
"logits/rejected": -0.884585976600647, |
|
"logps/chosen": -468.4187927246094, |
|
"logps/rejected": -541.056640625, |
|
"loss": 0.5474, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -1.829167366027832, |
|
"rewards/margins": 0.9327658414840698, |
|
"rewards/rejected": -2.7619330883026123, |
|
"step": 800 |
|
}, |
|
{ |
|
"epoch": 0.4187385501177702, |
|
"eval_logits/chosen": -0.8851308822631836, |
|
"eval_logits/rejected": -0.9115467667579651, |
|
"eval_logps/chosen": -455.87139892578125, |
|
"eval_logps/rejected": -529.6283569335938, |
|
"eval_loss": 0.4966377317905426, |
|
"eval_rewards/accuracies": 0.7670000195503235, |
|
"eval_rewards/chosen": -1.794780969619751, |
|
"eval_rewards/margins": 0.9688891768455505, |
|
"eval_rewards/rejected": -2.7636702060699463, |
|
"eval_runtime": 491.3021, |
|
"eval_samples_per_second": 4.071, |
|
"eval_steps_per_second": 0.254, |
|
"step": 800 |
|
}, |
|
{ |
|
"epoch": 0.4239727819942423, |
|
"grad_norm": 4.3302971642908465, |
|
"learning_rate": 3.56378672954129e-06, |
|
"logits/chosen": -0.8280191421508789, |
|
"logits/rejected": -0.8614367246627808, |
|
"logps/chosen": -439.85662841796875, |
|
"logps/rejected": -539.2572021484375, |
|
"loss": 0.5092, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -1.863586187362671, |
|
"rewards/margins": 0.8855009078979492, |
|
"rewards/rejected": -2.74908709526062, |
|
"step": 810 |
|
}, |
|
{ |
|
"epoch": 0.42920701387071447, |
|
"grad_norm": 3.1116409091724786, |
|
"learning_rate": 3.5222648293233806e-06, |
|
"logits/chosen": -0.8204169273376465, |
|
"logits/rejected": -0.8331009149551392, |
|
"logps/chosen": -489.24383544921875, |
|
"logps/rejected": -579.9151611328125, |
|
"loss": 0.5038, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.3326661586761475, |
|
"rewards/margins": 1.1430214643478394, |
|
"rewards/rejected": -3.4756877422332764, |
|
"step": 820 |
|
}, |
|
{ |
|
"epoch": 0.4344412457471866, |
|
"grad_norm": 3.2429890344778074, |
|
"learning_rate": 3.4804015009566573e-06, |
|
"logits/chosen": -0.7415329217910767, |
|
"logits/rejected": -0.7724164724349976, |
|
"logps/chosen": -455.18572998046875, |
|
"logps/rejected": -487.82733154296875, |
|
"loss": 0.5401, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -2.0002198219299316, |
|
"rewards/margins": 0.7170491218566895, |
|
"rewards/rejected": -2.717268705368042, |
|
"step": 830 |
|
}, |
|
{ |
|
"epoch": 0.4396754776236587, |
|
"grad_norm": 3.5668607410560975, |
|
"learning_rate": 3.4382107264527244e-06, |
|
"logits/chosen": -0.7589952945709229, |
|
"logits/rejected": -0.7982163429260254, |
|
"logps/chosen": -483.5228576660156, |
|
"logps/rejected": -528.97021484375, |
|
"loss": 0.4736, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": -1.7952258586883545, |
|
"rewards/margins": 1.0175402164459229, |
|
"rewards/rejected": -2.8127660751342773, |
|
"step": 840 |
|
}, |
|
{ |
|
"epoch": 0.44490970950013087, |
|
"grad_norm": 4.0501340588509365, |
|
"learning_rate": 3.3957065971875387e-06, |
|
"logits/chosen": -0.795210063457489, |
|
"logits/rejected": -0.8109579086303711, |
|
"logps/chosen": -458.65814208984375, |
|
"logps/rejected": -532.5260620117188, |
|
"loss": 0.4962, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -2.041184902191162, |
|
"rewards/margins": 1.0093342065811157, |
|
"rewards/rejected": -3.0505189895629883, |
|
"step": 850 |
|
}, |
|
{ |
|
"epoch": 0.45014394137660296, |
|
"grad_norm": 3.973745164078984, |
|
"learning_rate": 3.352903309194999e-06, |
|
"logits/chosen": -0.8046371340751648, |
|
"logits/rejected": -0.8593829274177551, |
|
"logps/chosen": -461.7774353027344, |
|
"logps/rejected": -595.0950927734375, |
|
"loss": 0.4721, |
|
"rewards/accuracies": 0.793749988079071, |
|
"rewards/chosen": -2.2889552116394043, |
|
"rewards/margins": 1.1978638172149658, |
|
"rewards/rejected": -3.4868195056915283, |
|
"step": 860 |
|
}, |
|
{ |
|
"epoch": 0.4553781732530751, |
|
"grad_norm": 2.672124703356638, |
|
"learning_rate": 3.309815158425591e-06, |
|
"logits/chosen": -0.7477758526802063, |
|
"logits/rejected": -0.7731175422668457, |
|
"logps/chosen": -472.6768493652344, |
|
"logps/rejected": -555.8510131835938, |
|
"loss": 0.5073, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -2.2288527488708496, |
|
"rewards/margins": 0.9096274375915527, |
|
"rewards/rejected": -3.1384804248809814, |
|
"step": 870 |
|
}, |
|
{ |
|
"epoch": 0.46061240512954726, |
|
"grad_norm": 2.903999411374589, |
|
"learning_rate": 3.266456535971654e-06, |
|
"logits/chosen": -0.864072322845459, |
|
"logits/rejected": -0.8836487531661987, |
|
"logps/chosen": -464.50115966796875, |
|
"logps/rejected": -530.8693237304688, |
|
"loss": 0.4929, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -2.0616631507873535, |
|
"rewards/margins": 0.824569821357727, |
|
"rewards/rejected": -2.88623309135437, |
|
"step": 880 |
|
}, |
|
{ |
|
"epoch": 0.46584663700601936, |
|
"grad_norm": 2.4287136163193974, |
|
"learning_rate": 3.2228419232608692e-06, |
|
"logits/chosen": -0.8057514429092407, |
|
"logits/rejected": -0.8173397183418274, |
|
"logps/chosen": -457.15032958984375, |
|
"logps/rejected": -542.5882568359375, |
|
"loss": 0.4903, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -1.8502060174942017, |
|
"rewards/margins": 0.9911335110664368, |
|
"rewards/rejected": -2.841339588165283, |
|
"step": 890 |
|
}, |
|
{ |
|
"epoch": 0.4710808688824915, |
|
"grad_norm": 2.8863413320590774, |
|
"learning_rate": 3.1789858872195888e-06, |
|
"logits/chosen": -0.771654486656189, |
|
"logits/rejected": -0.8108331561088562, |
|
"logps/chosen": -418.285888671875, |
|
"logps/rejected": -514.6083984375, |
|
"loss": 0.5246, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -1.6747478246688843, |
|
"rewards/margins": 0.8920180201530457, |
|
"rewards/rejected": -2.566765785217285, |
|
"step": 900 |
|
}, |
|
{ |
|
"epoch": 0.4710808688824915, |
|
"eval_logits/chosen": -0.7979649305343628, |
|
"eval_logits/rejected": -0.8138096928596497, |
|
"eval_logps/chosen": -429.2431335449219, |
|
"eval_logps/rejected": -507.4219055175781, |
|
"eval_loss": 0.49433061480522156, |
|
"eval_rewards/accuracies": 0.765999972820282, |
|
"eval_rewards/chosen": -1.5284981727600098, |
|
"eval_rewards/margins": 1.013107419013977, |
|
"eval_rewards/rejected": -2.5416059494018555, |
|
"eval_runtime": 490.1393, |
|
"eval_samples_per_second": 4.08, |
|
"eval_steps_per_second": 0.255, |
|
"step": 900 |
|
}, |
|
{ |
|
"epoch": 0.4763151007589636, |
|
"grad_norm": 2.884609247157233, |
|
"learning_rate": 3.1349030754075945e-06, |
|
"logits/chosen": -0.7577202320098877, |
|
"logits/rejected": -0.764434278011322, |
|
"logps/chosen": -435.4732360839844, |
|
"logps/rejected": -516.2320556640625, |
|
"loss": 0.5219, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -1.7493627071380615, |
|
"rewards/margins": 0.8683775067329407, |
|
"rewards/rejected": -2.6177401542663574, |
|
"step": 910 |
|
}, |
|
{ |
|
"epoch": 0.48154933263543576, |
|
"grad_norm": 2.1669192855886257, |
|
"learning_rate": 3.0906082111259313e-06, |
|
"logits/chosen": -0.8027156591415405, |
|
"logits/rejected": -0.8391119837760925, |
|
"logps/chosen": -479.461181640625, |
|
"logps/rejected": -578.0491943359375, |
|
"loss": 0.4802, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -2.1167943477630615, |
|
"rewards/margins": 1.2451462745666504, |
|
"rewards/rejected": -3.361940383911133, |
|
"step": 920 |
|
}, |
|
{ |
|
"epoch": 0.48678356451190785, |
|
"grad_norm": 3.304438899218671, |
|
"learning_rate": 3.046116088499449e-06, |
|
"logits/chosen": -0.8075677752494812, |
|
"logits/rejected": -0.818462073802948, |
|
"logps/chosen": -551.7511596679688, |
|
"logps/rejected": -617.6654052734375, |
|
"loss": 0.4831, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -2.702815532684326, |
|
"rewards/margins": 0.8609806895256042, |
|
"rewards/rejected": -3.563796281814575, |
|
"step": 930 |
|
}, |
|
{ |
|
"epoch": 0.49201779638838, |
|
"grad_norm": 5.162139925744972, |
|
"learning_rate": 3.0014415675356813e-06, |
|
"logits/chosen": -0.8480289578437805, |
|
"logits/rejected": -0.8807282447814941, |
|
"logps/chosen": -580.1326904296875, |
|
"logps/rejected": -658.1190795898438, |
|
"loss": 0.451, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -2.9515488147735596, |
|
"rewards/margins": 1.0807795524597168, |
|
"rewards/rejected": -4.032328128814697, |
|
"step": 940 |
|
}, |
|
{ |
|
"epoch": 0.49725202826485215, |
|
"grad_norm": 4.270308518724467, |
|
"learning_rate": 2.9565995691617242e-06, |
|
"logits/chosen": -0.8305244445800781, |
|
"logits/rejected": -0.8603514432907104, |
|
"logps/chosen": -577.685302734375, |
|
"logps/rejected": -670.2604370117188, |
|
"loss": 0.4688, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.654804229736328, |
|
"rewards/margins": 1.3251168727874756, |
|
"rewards/rejected": -3.9799208641052246, |
|
"step": 950 |
|
}, |
|
{ |
|
"epoch": 0.5024862601413242, |
|
"grad_norm": 3.9112848274397374, |
|
"learning_rate": 2.9116050702407706e-06, |
|
"logits/chosen": -0.8263424634933472, |
|
"logits/rejected": -0.8838019371032715, |
|
"logps/chosen": -500.05841064453125, |
|
"logps/rejected": -590.2412109375, |
|
"loss": 0.5068, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.197012424468994, |
|
"rewards/margins": 1.2063963413238525, |
|
"rewards/rejected": -3.4034087657928467, |
|
"step": 960 |
|
}, |
|
{ |
|
"epoch": 0.5077204920177963, |
|
"grad_norm": 2.6527116057322813, |
|
"learning_rate": 2.8664730985699537e-06, |
|
"logits/chosen": -0.8739676475524902, |
|
"logits/rejected": -0.8983599543571472, |
|
"logps/chosen": -471.83978271484375, |
|
"logps/rejected": -584.647216796875, |
|
"loss": 0.4578, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.0771896839141846, |
|
"rewards/margins": 1.0904505252838135, |
|
"rewards/rejected": -3.167639970779419, |
|
"step": 970 |
|
}, |
|
{ |
|
"epoch": 0.5129547238942685, |
|
"grad_norm": 3.0654272233325806, |
|
"learning_rate": 2.8212187278611907e-06, |
|
"logits/chosen": -0.8305708765983582, |
|
"logits/rejected": -0.8489472270011902, |
|
"logps/chosen": -484.65679931640625, |
|
"logps/rejected": -540.0225219726562, |
|
"loss": 0.4925, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -1.8167269229888916, |
|
"rewards/margins": 0.965971827507019, |
|
"rewards/rejected": -2.7826988697052, |
|
"step": 980 |
|
}, |
|
{ |
|
"epoch": 0.5181889557707406, |
|
"grad_norm": 2.8308862425981896, |
|
"learning_rate": 2.7758570727066843e-06, |
|
"logits/chosen": -0.8484777212142944, |
|
"logits/rejected": -0.861672580242157, |
|
"logps/chosen": -470.0426330566406, |
|
"logps/rejected": -552.4635620117188, |
|
"loss": 0.5011, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -1.991310715675354, |
|
"rewards/margins": 1.036561369895935, |
|
"rewards/rejected": -3.027872085571289, |
|
"step": 990 |
|
}, |
|
{ |
|
"epoch": 0.5234231876472127, |
|
"grad_norm": 3.1475704256823818, |
|
"learning_rate": 2.730403283530767e-06, |
|
"logits/chosen": -0.7995238900184631, |
|
"logits/rejected": -0.8208199739456177, |
|
"logps/chosen": -514.13525390625, |
|
"logps/rejected": -616.7322387695312, |
|
"loss": 0.4635, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -2.589371681213379, |
|
"rewards/margins": 1.151365876197815, |
|
"rewards/rejected": -3.7407374382019043, |
|
"step": 1000 |
|
}, |
|
{ |
|
"epoch": 0.5234231876472127, |
|
"eval_logits/chosen": -0.8520967364311218, |
|
"eval_logits/rejected": -0.871288001537323, |
|
"eval_logps/chosen": -558.1610107421875, |
|
"eval_logps/rejected": -656.6334228515625, |
|
"eval_loss": 0.4907907247543335, |
|
"eval_rewards/accuracies": 0.7630000114440918, |
|
"eval_rewards/chosen": -2.817676544189453, |
|
"eval_rewards/margins": 1.2160439491271973, |
|
"eval_rewards/rejected": -4.03372049331665, |
|
"eval_runtime": 490.4932, |
|
"eval_samples_per_second": 4.078, |
|
"eval_steps_per_second": 0.255, |
|
"step": 1000 |
|
}, |
|
{ |
|
"epoch": 0.528657419523685, |
|
"grad_norm": 2.8934222921049226, |
|
"learning_rate": 2.6848725415297888e-06, |
|
"logits/chosen": -0.8423219919204712, |
|
"logits/rejected": -0.8691143989562988, |
|
"logps/chosen": -526.5357666015625, |
|
"logps/rejected": -626.977294921875, |
|
"loss": 0.4864, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -2.859957456588745, |
|
"rewards/margins": 1.0318825244903564, |
|
"rewards/rejected": -3.8918404579162598, |
|
"step": 1010 |
|
}, |
|
{ |
|
"epoch": 0.533891651400157, |
|
"grad_norm": 4.072441147880593, |
|
"learning_rate": 2.639280053601719e-06, |
|
"logits/chosen": -0.7951158285140991, |
|
"logits/rejected": -0.8444026708602905, |
|
"logps/chosen": -551.2407836914062, |
|
"logps/rejected": -624.159912109375, |
|
"loss": 0.4753, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -2.523639678955078, |
|
"rewards/margins": 1.181814193725586, |
|
"rewards/rejected": -3.7054543495178223, |
|
"step": 1020 |
|
}, |
|
{ |
|
"epoch": 0.5391258832766291, |
|
"grad_norm": 2.288139837580109, |
|
"learning_rate": 2.59364104726716e-06, |
|
"logits/chosen": -0.7849830389022827, |
|
"logits/rejected": -0.7941724061965942, |
|
"logps/chosen": -558.4784545898438, |
|
"logps/rejected": -631.0448608398438, |
|
"loss": 0.4813, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -2.9077861309051514, |
|
"rewards/margins": 1.099591851234436, |
|
"rewards/rejected": -4.007378101348877, |
|
"step": 1030 |
|
}, |
|
{ |
|
"epoch": 0.5443601151531012, |
|
"grad_norm": 2.255027947939241, |
|
"learning_rate": 2.547970765583491e-06, |
|
"logits/chosen": -0.7766658663749695, |
|
"logits/rejected": -0.7682461142539978, |
|
"logps/chosen": -551.2001953125, |
|
"logps/rejected": -626.3408203125, |
|
"loss": 0.508, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -2.647867202758789, |
|
"rewards/margins": 0.965995192527771, |
|
"rewards/rejected": -3.6138622760772705, |
|
"step": 1040 |
|
}, |
|
{ |
|
"epoch": 0.5495943470295734, |
|
"grad_norm": 3.365620498364263, |
|
"learning_rate": 2.502284462053799e-06, |
|
"logits/chosen": -0.7241968512535095, |
|
"logits/rejected": -0.7473636269569397, |
|
"logps/chosen": -527.13427734375, |
|
"logps/rejected": -617.5802612304688, |
|
"loss": 0.4658, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.2644190788269043, |
|
"rewards/margins": 1.2528817653656006, |
|
"rewards/rejected": -3.517301082611084, |
|
"step": 1050 |
|
}, |
|
{ |
|
"epoch": 0.5548285789060455, |
|
"grad_norm": 4.206518455943664, |
|
"learning_rate": 2.456597395532338e-06, |
|
"logits/chosen": -0.7641677856445312, |
|
"logits/rejected": -0.8080291748046875, |
|
"logps/chosen": -470.62872314453125, |
|
"logps/rejected": -533.24560546875, |
|
"loss": 0.4611, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -2.032501220703125, |
|
"rewards/margins": 1.0605053901672363, |
|
"rewards/rejected": -3.0930066108703613, |
|
"step": 1060 |
|
}, |
|
{ |
|
"epoch": 0.5600628107825176, |
|
"grad_norm": 2.230846159297882, |
|
"learning_rate": 2.4109248251281953e-06, |
|
"logits/chosen": -0.7514528632164001, |
|
"logits/rejected": -0.775009036064148, |
|
"logps/chosen": -500.794921875, |
|
"logps/rejected": -553.78466796875, |
|
"loss": 0.4948, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -2.0267245769500732, |
|
"rewards/margins": 1.0498759746551514, |
|
"rewards/rejected": -3.0766005516052246, |
|
"step": 1070 |
|
}, |
|
{ |
|
"epoch": 0.5652970426589898, |
|
"grad_norm": 2.9850161302908567, |
|
"learning_rate": 2.365282005108875e-06, |
|
"logits/chosen": -0.7392014265060425, |
|
"logits/rejected": -0.7644953727722168, |
|
"logps/chosen": -484.65216064453125, |
|
"logps/rejected": -520.8948974609375, |
|
"loss": 0.5131, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -2.1898112297058105, |
|
"rewards/margins": 0.9680379629135132, |
|
"rewards/rejected": -3.1578493118286133, |
|
"step": 1080 |
|
}, |
|
{ |
|
"epoch": 0.5705312745354619, |
|
"grad_norm": 4.871043269302411, |
|
"learning_rate": 2.319684179805491e-06, |
|
"logits/chosen": -0.7259557247161865, |
|
"logits/rejected": -0.7469737529754639, |
|
"logps/chosen": -513.62353515625, |
|
"logps/rejected": -555.1145629882812, |
|
"loss": 0.5266, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -2.5438778400421143, |
|
"rewards/margins": 0.932669997215271, |
|
"rewards/rejected": -3.476547956466675, |
|
"step": 1090 |
|
}, |
|
{ |
|
"epoch": 0.575765506411934, |
|
"grad_norm": 2.9604109906028677, |
|
"learning_rate": 2.2741465785212905e-06, |
|
"logits/chosen": -0.7221305966377258, |
|
"logits/rejected": -0.7538542747497559, |
|
"logps/chosen": -503.03143310546875, |
|
"logps/rejected": -606.3543090820312, |
|
"loss": 0.4856, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -2.407829761505127, |
|
"rewards/margins": 0.9688541293144226, |
|
"rewards/rejected": -3.3766837120056152, |
|
"step": 1100 |
|
}, |
|
{ |
|
"epoch": 0.575765506411934, |
|
"eval_logits/chosen": -0.7912909984588623, |
|
"eval_logits/rejected": -0.8044111728668213, |
|
"eval_logps/chosen": -512.9990234375, |
|
"eval_logps/rejected": -602.4694213867188, |
|
"eval_loss": 0.4817214012145996, |
|
"eval_rewards/accuracies": 0.7720000147819519, |
|
"eval_rewards/chosen": -2.3660569190979004, |
|
"eval_rewards/margins": 1.1260240077972412, |
|
"eval_rewards/rejected": -3.4920809268951416, |
|
"eval_runtime": 490.5656, |
|
"eval_samples_per_second": 4.077, |
|
"eval_steps_per_second": 0.255, |
|
"step": 1100 |
|
}, |
|
{ |
|
"epoch": 0.5809997382884062, |
|
"grad_norm": 4.040674423079137, |
|
"learning_rate": 2.2286844104451848e-06, |
|
"logits/chosen": -0.7596691250801086, |
|
"logits/rejected": -0.7672920227050781, |
|
"logps/chosen": -506.1031799316406, |
|
"logps/rejected": -615.0109252929688, |
|
"loss": 0.51, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.4000015258789062, |
|
"rewards/margins": 1.183712363243103, |
|
"rewards/rejected": -3.583714008331299, |
|
"step": 1110 |
|
}, |
|
{ |
|
"epoch": 0.5862339701648783, |
|
"grad_norm": 3.766321110228989, |
|
"learning_rate": 2.183312859572008e-06, |
|
"logits/chosen": -0.7806628346443176, |
|
"logits/rejected": -0.7962976098060608, |
|
"logps/chosen": -550.5853271484375, |
|
"logps/rejected": -597.5955200195312, |
|
"loss": 0.5075, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -2.502775192260742, |
|
"rewards/margins": 0.9967296719551086, |
|
"rewards/rejected": -3.499504804611206, |
|
"step": 1120 |
|
}, |
|
{ |
|
"epoch": 0.5914682020413504, |
|
"grad_norm": 3.8765385487802715, |
|
"learning_rate": 2.1380470796311843e-06, |
|
"logits/chosen": -0.7329021692276001, |
|
"logits/rejected": -0.716983437538147, |
|
"logps/chosen": -548.9044799804688, |
|
"logps/rejected": -640.8181762695312, |
|
"loss": 0.4782, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -2.5259509086608887, |
|
"rewards/margins": 1.2471174001693726, |
|
"rewards/rejected": -3.77306866645813, |
|
"step": 1130 |
|
}, |
|
{ |
|
"epoch": 0.5967024339178225, |
|
"grad_norm": 3.776816209238374, |
|
"learning_rate": 2.092902189025507e-06, |
|
"logits/chosen": -0.7345478534698486, |
|
"logits/rejected": -0.765870213508606, |
|
"logps/chosen": -567.6248168945312, |
|
"logps/rejected": -644.7882690429688, |
|
"loss": 0.4409, |
|
"rewards/accuracies": 0.793749988079071, |
|
"rewards/chosen": -2.6834120750427246, |
|
"rewards/margins": 1.2605786323547363, |
|
"rewards/rejected": -3.943991184234619, |
|
"step": 1140 |
|
}, |
|
{ |
|
"epoch": 0.6019366657942947, |
|
"grad_norm": 2.4633069424861476, |
|
"learning_rate": 2.0478932657817105e-06, |
|
"logits/chosen": -0.7708589434623718, |
|
"logits/rejected": -0.7781729698181152, |
|
"logps/chosen": -546.5011596679688, |
|
"logps/rejected": -616.3358154296875, |
|
"loss": 0.4785, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -2.6856911182403564, |
|
"rewards/margins": 1.0351371765136719, |
|
"rewards/rejected": -3.7208282947540283, |
|
"step": 1150 |
|
}, |
|
{ |
|
"epoch": 0.6071708976707668, |
|
"grad_norm": 2.9067779299636363, |
|
"learning_rate": 2.0030353425145376e-06, |
|
"logits/chosen": -0.7892950773239136, |
|
"logits/rejected": -0.7964081764221191, |
|
"logps/chosen": -454.22735595703125, |
|
"logps/rejected": -548.8640747070312, |
|
"loss": 0.551, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -2.2731716632843018, |
|
"rewards/margins": 1.0861868858337402, |
|
"rewards/rejected": -3.359358549118042, |
|
"step": 1160 |
|
}, |
|
{ |
|
"epoch": 0.6124051295472389, |
|
"grad_norm": 3.0126218778741816, |
|
"learning_rate": 1.958343401405964e-06, |
|
"logits/chosen": -0.7989486455917358, |
|
"logits/rejected": -0.8354769945144653, |
|
"logps/chosen": -471.26971435546875, |
|
"logps/rejected": -576.9542236328125, |
|
"loss": 0.515, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -1.9779512882232666, |
|
"rewards/margins": 1.1961392164230347, |
|
"rewards/rejected": -3.1740903854370117, |
|
"step": 1170 |
|
}, |
|
{ |
|
"epoch": 0.6176393614237111, |
|
"grad_norm": 2.8356127345530298, |
|
"learning_rate": 1.9138323692012734e-06, |
|
"logits/chosen": -0.7760749459266663, |
|
"logits/rejected": -0.8243614435195923, |
|
"logps/chosen": -477.05255126953125, |
|
"logps/rejected": -567.8319091796875, |
|
"loss": 0.4482, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -1.9603474140167236, |
|
"rewards/margins": 1.083135962486267, |
|
"rewards/rejected": -3.0434834957122803, |
|
"step": 1180 |
|
}, |
|
{ |
|
"epoch": 0.6228735933001832, |
|
"grad_norm": 2.653741359724002, |
|
"learning_rate": 1.8695171122236443e-06, |
|
"logits/chosen": -0.7773482799530029, |
|
"logits/rejected": -0.8091195821762085, |
|
"logps/chosen": -534.4111328125, |
|
"logps/rejected": -603.28564453125, |
|
"loss": 0.4399, |
|
"rewards/accuracies": 0.84375, |
|
"rewards/chosen": -2.1728804111480713, |
|
"rewards/margins": 1.205948829650879, |
|
"rewards/rejected": -3.37882924079895, |
|
"step": 1190 |
|
}, |
|
{ |
|
"epoch": 0.6281078251766553, |
|
"grad_norm": 3.666686057880921, |
|
"learning_rate": 1.8254124314089225e-06, |
|
"logits/chosen": -0.7455651760101318, |
|
"logits/rejected": -0.7562496066093445, |
|
"logps/chosen": -496.6683044433594, |
|
"logps/rejected": -561.0657958984375, |
|
"loss": 0.5013, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -2.1546523571014404, |
|
"rewards/margins": 0.8877624273300171, |
|
"rewards/rejected": -3.042415142059326, |
|
"step": 1200 |
|
}, |
|
{ |
|
"epoch": 0.6281078251766553, |
|
"eval_logits/chosen": -0.7745435237884521, |
|
"eval_logits/rejected": -0.7890085577964783, |
|
"eval_logps/chosen": -488.01080322265625, |
|
"eval_logps/rejected": -582.3287353515625, |
|
"eval_loss": 0.4859620928764343, |
|
"eval_rewards/accuracies": 0.7720000147819519, |
|
"eval_rewards/chosen": -2.1161751747131348, |
|
"eval_rewards/margins": 1.1744980812072754, |
|
"eval_rewards/rejected": -3.29067325592041, |
|
"eval_runtime": 492.7641, |
|
"eval_samples_per_second": 4.059, |
|
"eval_steps_per_second": 0.254, |
|
"step": 1200 |
|
}, |
|
{ |
|
"epoch": 0.6333420570531274, |
|
"grad_norm": 4.445062813407665, |
|
"learning_rate": 1.781533057362221e-06, |
|
"logits/chosen": -0.7981818914413452, |
|
"logits/rejected": -0.8196622729301453, |
|
"logps/chosen": -482.60601806640625, |
|
"logps/rejected": -558.7507934570312, |
|
"loss": 0.4991, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -2.063744068145752, |
|
"rewards/margins": 1.000634789466858, |
|
"rewards/rejected": -3.0643787384033203, |
|
"step": 1210 |
|
}, |
|
{ |
|
"epoch": 0.6385762889295996, |
|
"grad_norm": 4.507009476169383, |
|
"learning_rate": 1.7378936454380277e-06, |
|
"logits/chosen": -0.7655137181282043, |
|
"logits/rejected": -0.7727285623550415, |
|
"logps/chosen": -491.4285583496094, |
|
"logps/rejected": -586.7752685546875, |
|
"loss": 0.4859, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.225255012512207, |
|
"rewards/margins": 1.2578017711639404, |
|
"rewards/rejected": -3.4830570220947266, |
|
"step": 1220 |
|
}, |
|
{ |
|
"epoch": 0.6438105208060717, |
|
"grad_norm": 2.769661114858915, |
|
"learning_rate": 1.6945087708454273e-06, |
|
"logits/chosen": -0.7237203121185303, |
|
"logits/rejected": -0.752585768699646, |
|
"logps/chosen": -524.2711791992188, |
|
"logps/rejected": -591.5977783203125, |
|
"loss": 0.4308, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.2781286239624023, |
|
"rewards/margins": 1.1484158039093018, |
|
"rewards/rejected": -3.426544666290283, |
|
"step": 1230 |
|
}, |
|
{ |
|
"epoch": 0.6490447526825438, |
|
"grad_norm": 2.8409702645316486, |
|
"learning_rate": 1.651392923780105e-06, |
|
"logits/chosen": -0.7248971462249756, |
|
"logits/rejected": -0.7399358153343201, |
|
"logps/chosen": -502.482666015625, |
|
"logps/rejected": -614.2011108398438, |
|
"loss": 0.4665, |
|
"rewards/accuracies": 0.8125, |
|
"rewards/chosen": -2.312042713165283, |
|
"rewards/margins": 1.1948063373565674, |
|
"rewards/rejected": -3.5068485736846924, |
|
"step": 1240 |
|
}, |
|
{ |
|
"epoch": 0.654278984559016, |
|
"grad_norm": 2.5610614385353614, |
|
"learning_rate": 1.608560504584737e-06, |
|
"logits/chosen": -0.7505759000778198, |
|
"logits/rejected": -0.748982846736908, |
|
"logps/chosen": -497.2088317871094, |
|
"logps/rejected": -608.0076904296875, |
|
"loss": 0.4391, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -2.297612428665161, |
|
"rewards/margins": 1.178468942642212, |
|
"rewards/rejected": -3.476081132888794, |
|
"step": 1250 |
|
}, |
|
{ |
|
"epoch": 0.6595132164354881, |
|
"grad_norm": 2.9288478965975204, |
|
"learning_rate": 1.5660258189393945e-06, |
|
"logits/chosen": -0.7709019780158997, |
|
"logits/rejected": -0.7804166674613953, |
|
"logps/chosen": -484.7782287597656, |
|
"logps/rejected": -561.656494140625, |
|
"loss": 0.4529, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -2.0165915489196777, |
|
"rewards/margins": 1.224448323249817, |
|
"rewards/rejected": -3.241039752960205, |
|
"step": 1260 |
|
}, |
|
{ |
|
"epoch": 0.6647474483119602, |
|
"grad_norm": 3.3843415697325123, |
|
"learning_rate": 1.5238030730835578e-06, |
|
"logits/chosen": -0.7160421013832092, |
|
"logits/rejected": -0.7430087327957153, |
|
"logps/chosen": -469.3851623535156, |
|
"logps/rejected": -589.1715698242188, |
|
"loss": 0.4304, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -2.223741292953491, |
|
"rewards/margins": 1.258303165435791, |
|
"rewards/rejected": -3.4820446968078613, |
|
"step": 1270 |
|
}, |
|
{ |
|
"epoch": 0.6699816801884323, |
|
"grad_norm": 4.31798859648472, |
|
"learning_rate": 1.4819063690713565e-06, |
|
"logits/chosen": -0.7581356167793274, |
|
"logits/rejected": -0.7594733238220215, |
|
"logps/chosen": -471.11822509765625, |
|
"logps/rejected": -587.6780395507812, |
|
"loss": 0.4618, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -2.2457804679870605, |
|
"rewards/margins": 1.2690086364746094, |
|
"rewards/rejected": -3.514788866043091, |
|
"step": 1280 |
|
}, |
|
{ |
|
"epoch": 0.6752159120649045, |
|
"grad_norm": 2.9043056997624785, |
|
"learning_rate": 1.4403497000615885e-06, |
|
"logits/chosen": -0.729811429977417, |
|
"logits/rejected": -0.7537652254104614, |
|
"logps/chosen": -517.2977294921875, |
|
"logps/rejected": -622.2907104492188, |
|
"loss": 0.5079, |
|
"rewards/accuracies": 0.7437499761581421, |
|
"rewards/chosen": -2.318469524383545, |
|
"rewards/margins": 1.2079073190689087, |
|
"rewards/rejected": -3.5263772010803223, |
|
"step": 1290 |
|
}, |
|
{ |
|
"epoch": 0.6804501439413766, |
|
"grad_norm": 3.476254076191995, |
|
"learning_rate": 1.3991469456441273e-06, |
|
"logits/chosen": -0.7037397623062134, |
|
"logits/rejected": -0.7578923106193542, |
|
"logps/chosen": -535.3514404296875, |
|
"logps/rejected": -628.6389770507812, |
|
"loss": 0.4497, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.396596670150757, |
|
"rewards/margins": 1.2228056192398071, |
|
"rewards/rejected": -3.6194019317626953, |
|
"step": 1300 |
|
}, |
|
{ |
|
"epoch": 0.6804501439413766, |
|
"eval_logits/chosen": -0.7940347790718079, |
|
"eval_logits/rejected": -0.8095559477806091, |
|
"eval_logps/chosen": -524.7894897460938, |
|
"eval_logps/rejected": -626.9693603515625, |
|
"eval_loss": 0.485016793012619, |
|
"eval_rewards/accuracies": 0.7730000019073486, |
|
"eval_rewards/chosen": -2.483961820602417, |
|
"eval_rewards/margins": 1.2531172037124634, |
|
"eval_rewards/rejected": -3.73707914352417, |
|
"eval_runtime": 490.5183, |
|
"eval_samples_per_second": 4.077, |
|
"eval_steps_per_second": 0.255, |
|
"step": 1300 |
|
}, |
|
{ |
|
"epoch": 0.6856843758178487, |
|
"grad_norm": 4.224665538714578, |
|
"learning_rate": 1.3583118672042441e-06, |
|
"logits/chosen": -0.7764405608177185, |
|
"logits/rejected": -0.7887049913406372, |
|
"logps/chosen": -516.7277221679688, |
|
"logps/rejected": -632.888427734375, |
|
"loss": 0.472, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -2.473681926727295, |
|
"rewards/margins": 1.352980136871338, |
|
"rewards/rejected": -3.8266615867614746, |
|
"step": 1310 |
|
}, |
|
{ |
|
"epoch": 0.6909186076943209, |
|
"grad_norm": 4.563976167233384, |
|
"learning_rate": 1.3178581033264218e-06, |
|
"logits/chosen": -0.7718938589096069, |
|
"logits/rejected": -0.7976632714271545, |
|
"logps/chosen": -519.279541015625, |
|
"logps/rejected": -601.4541015625, |
|
"loss": 0.4487, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -2.553548812866211, |
|
"rewards/margins": 1.0802887678146362, |
|
"rewards/rejected": -3.633837938308716, |
|
"step": 1320 |
|
}, |
|
{ |
|
"epoch": 0.696152839570793, |
|
"grad_norm": 4.184522446421226, |
|
"learning_rate": 1.2777991652391757e-06, |
|
"logits/chosen": -0.7361895442008972, |
|
"logits/rejected": -0.7566362619400024, |
|
"logps/chosen": -527.201416015625, |
|
"logps/rejected": -645.9843139648438, |
|
"loss": 0.4656, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.564059257507324, |
|
"rewards/margins": 1.190525770187378, |
|
"rewards/rejected": -3.7545852661132812, |
|
"step": 1330 |
|
}, |
|
{ |
|
"epoch": 0.7013870714472651, |
|
"grad_norm": 4.166233355017187, |
|
"learning_rate": 1.2381484323024178e-06, |
|
"logits/chosen": -0.713969349861145, |
|
"logits/rejected": -0.7467511892318726, |
|
"logps/chosen": -518.2627563476562, |
|
"logps/rejected": -595.9432373046875, |
|
"loss": 0.4851, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.4644479751586914, |
|
"rewards/margins": 1.2395137548446655, |
|
"rewards/rejected": -3.7039618492126465, |
|
"step": 1340 |
|
}, |
|
{ |
|
"epoch": 0.7066213033237373, |
|
"grad_norm": 3.865579244226054, |
|
"learning_rate": 1.1989191475388518e-06, |
|
"logits/chosen": -0.75743567943573, |
|
"logits/rejected": -0.7837706804275513, |
|
"logps/chosen": -509.4811096191406, |
|
"logps/rejected": -594.09033203125, |
|
"loss": 0.4915, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -2.323971748352051, |
|
"rewards/margins": 1.0227216482162476, |
|
"rewards/rejected": -3.346693515777588, |
|
"step": 1350 |
|
}, |
|
{ |
|
"epoch": 0.7118555352002094, |
|
"grad_norm": 2.652467671657092, |
|
"learning_rate": 1.160124413210918e-06, |
|
"logits/chosen": -0.7264483571052551, |
|
"logits/rejected": -0.7697917222976685, |
|
"logps/chosen": -533.6471557617188, |
|
"logps/rejected": -609.0975341796875, |
|
"loss": 0.4969, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.2726407051086426, |
|
"rewards/margins": 1.1175401210784912, |
|
"rewards/rejected": -3.3901805877685547, |
|
"step": 1360 |
|
}, |
|
{ |
|
"epoch": 0.7170897670766815, |
|
"grad_norm": 2.5577134603401506, |
|
"learning_rate": 1.1217771864447396e-06, |
|
"logits/chosen": -0.8089788556098938, |
|
"logits/rejected": -0.8140621185302734, |
|
"logps/chosen": -544.7674560546875, |
|
"logps/rejected": -623.3822631835938, |
|
"loss": 0.4132, |
|
"rewards/accuracies": 0.793749988079071, |
|
"rewards/chosen": -2.3746323585510254, |
|
"rewards/margins": 1.3340243101119995, |
|
"rewards/rejected": -3.7086567878723145, |
|
"step": 1370 |
|
}, |
|
{ |
|
"epoch": 0.7223239989531536, |
|
"grad_norm": 3.219449421326901, |
|
"learning_rate": 1.08389027490255e-06, |
|
"logits/chosen": -0.7905744314193726, |
|
"logits/rejected": -0.7953212857246399, |
|
"logps/chosen": -490.263916015625, |
|
"logps/rejected": -593.0517578125, |
|
"loss": 0.4631, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.2938039302825928, |
|
"rewards/margins": 1.0697680711746216, |
|
"rewards/rejected": -3.363571882247925, |
|
"step": 1380 |
|
}, |
|
{ |
|
"epoch": 0.7275582308296258, |
|
"grad_norm": 2.9190971855424936, |
|
"learning_rate": 1.046476332505036e-06, |
|
"logits/chosen": -0.736000657081604, |
|
"logits/rejected": -0.7762190103530884, |
|
"logps/chosen": -537.1323852539062, |
|
"logps/rejected": -622.3277587890625, |
|
"loss": 0.4459, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.433082103729248, |
|
"rewards/margins": 1.1599032878875732, |
|
"rewards/rejected": -3.592985153198242, |
|
"step": 1390 |
|
}, |
|
{ |
|
"epoch": 0.7327924627060979, |
|
"grad_norm": 3.087299031916978, |
|
"learning_rate": 1.0095478552050348e-06, |
|
"logits/chosen": -0.7585629224777222, |
|
"logits/rejected": -0.7968350648880005, |
|
"logps/chosen": -505.4625549316406, |
|
"logps/rejected": -628.9778442382812, |
|
"loss": 0.4734, |
|
"rewards/accuracies": 0.8062499761581421, |
|
"rewards/chosen": -2.4186644554138184, |
|
"rewards/margins": 1.282457947731018, |
|
"rewards/rejected": -3.701122283935547, |
|
"step": 1400 |
|
}, |
|
{ |
|
"epoch": 0.7327924627060979, |
|
"eval_logits/chosen": -0.7989752888679504, |
|
"eval_logits/rejected": -0.8147993087768555, |
|
"eval_logps/chosen": -491.04962158203125, |
|
"eval_logps/rejected": -590.251953125, |
|
"eval_loss": 0.4832901060581207, |
|
"eval_rewards/accuracies": 0.7739999890327454, |
|
"eval_rewards/chosen": -2.1465635299682617, |
|
"eval_rewards/margins": 1.223341703414917, |
|
"eval_rewards/rejected": -3.369905471801758, |
|
"eval_runtime": 492.7006, |
|
"eval_samples_per_second": 4.059, |
|
"eval_steps_per_second": 0.254, |
|
"step": 1400 |
|
}, |
|
{ |
|
"epoch": 0.73802669458257, |
|
"grad_norm": 3.5157604346203226, |
|
"learning_rate": 9.731171768139808e-07, |
|
"logits/chosen": -0.7569630146026611, |
|
"logits/rejected": -0.7974756956100464, |
|
"logps/chosen": -481.45477294921875, |
|
"logps/rejected": -610.00146484375, |
|
"loss": 0.5139, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -2.1769986152648926, |
|
"rewards/margins": 1.2288345098495483, |
|
"rewards/rejected": -3.4058330059051514, |
|
"step": 1410 |
|
}, |
|
{ |
|
"epoch": 0.7432609264590422, |
|
"grad_norm": 3.358269083637402, |
|
"learning_rate": 9.371964648825221e-07, |
|
"logits/chosen": -0.7507213950157166, |
|
"logits/rejected": -0.784144401550293, |
|
"logps/chosen": -489.03515625, |
|
"logps/rejected": -545.8272705078125, |
|
"loss": 0.5023, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -2.055467128753662, |
|
"rewards/margins": 0.9931677579879761, |
|
"rewards/rejected": -3.0486350059509277, |
|
"step": 1420 |
|
}, |
|
{ |
|
"epoch": 0.7484951583355143, |
|
"grad_norm": 3.450390480039677, |
|
"learning_rate": 9.017977166366445e-07, |
|
"logits/chosen": -0.7912155985832214, |
|
"logits/rejected": -0.8058408498764038, |
|
"logps/chosen": -509.6456604003906, |
|
"logps/rejected": -619.4719848632812, |
|
"loss": 0.4582, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -2.1829304695129395, |
|
"rewards/margins": 1.2820956707000732, |
|
"rewards/rejected": -3.4650261402130127, |
|
"step": 1430 |
|
}, |
|
{ |
|
"epoch": 0.7537293902119864, |
|
"grad_norm": 3.9019374901211066, |
|
"learning_rate": 8.669327549707096e-07, |
|
"logits/chosen": -0.8170151710510254, |
|
"logits/rejected": -0.8503853678703308, |
|
"logps/chosen": -525.7236328125, |
|
"logps/rejected": -609.9266967773438, |
|
"loss": 0.4976, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -2.2434840202331543, |
|
"rewards/margins": 0.9734998941421509, |
|
"rewards/rejected": -3.2169833183288574, |
|
"step": 1440 |
|
}, |
|
{ |
|
"epoch": 0.7589636220884585, |
|
"grad_norm": 3.1069704561124567, |
|
"learning_rate": 8.326132244986932e-07, |
|
"logits/chosen": -0.8235033750534058, |
|
"logits/rejected": -0.8337501287460327, |
|
"logps/chosen": -507.8706970214844, |
|
"logps/rejected": -617.8888549804688, |
|
"loss": 0.4459, |
|
"rewards/accuracies": 0.793749988079071, |
|
"rewards/chosen": -2.2898011207580566, |
|
"rewards/margins": 1.271906852722168, |
|
"rewards/rejected": -3.5617077350616455, |
|
"step": 1450 |
|
}, |
|
{ |
|
"epoch": 0.7641978539649307, |
|
"grad_norm": 2.5011020359087137, |
|
"learning_rate": 7.988505876649863e-07, |
|
"logits/chosen": -0.7653478384017944, |
|
"logits/rejected": -0.8202483057975769, |
|
"logps/chosen": -471.88299560546875, |
|
"logps/rejected": -579.7264404296875, |
|
"loss": 0.475, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -2.1682538986206055, |
|
"rewards/margins": 1.1780481338500977, |
|
"rewards/rejected": -3.346302032470703, |
|
"step": 1460 |
|
}, |
|
{ |
|
"epoch": 0.7694320858414028, |
|
"grad_norm": 2.063720356926022, |
|
"learning_rate": 7.656561209160248e-07, |
|
"logits/chosen": -0.7891589999198914, |
|
"logits/rejected": -0.8133258819580078, |
|
"logps/chosen": -516.5853271484375, |
|
"logps/rejected": -602.1696166992188, |
|
"loss": 0.467, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -2.344850540161133, |
|
"rewards/margins": 1.223329782485962, |
|
"rewards/rejected": -3.5681800842285156, |
|
"step": 1470 |
|
}, |
|
{ |
|
"epoch": 0.7746663177178749, |
|
"grad_norm": 3.4450781896535694, |
|
"learning_rate": 7.330409109340563e-07, |
|
"logits/chosen": -0.8023967742919922, |
|
"logits/rejected": -0.8343321084976196, |
|
"logps/chosen": -532.4755859375, |
|
"logps/rejected": -600.9357299804688, |
|
"loss": 0.4975, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -2.426600217819214, |
|
"rewards/margins": 1.1944777965545654, |
|
"rewards/rejected": -3.6210780143737793, |
|
"step": 1480 |
|
}, |
|
{ |
|
"epoch": 0.7799005495943471, |
|
"grad_norm": 4.640657971253548, |
|
"learning_rate": 7.010158509342682e-07, |
|
"logits/chosen": -0.7905557155609131, |
|
"logits/rejected": -0.8068881034851074, |
|
"logps/chosen": -546.9576416015625, |
|
"logps/rejected": -667.5574340820312, |
|
"loss": 0.483, |
|
"rewards/accuracies": 0.71875, |
|
"rewards/chosen": -2.799778699874878, |
|
"rewards/margins": 1.137909173965454, |
|
"rewards/rejected": -3.937687635421753, |
|
"step": 1490 |
|
}, |
|
{ |
|
"epoch": 0.7851347814708192, |
|
"grad_norm": 8.807489938353271, |
|
"learning_rate": 6.695916370265529e-07, |
|
"logits/chosen": -0.7907344698905945, |
|
"logits/rejected": -0.8382173776626587, |
|
"logps/chosen": -507.33575439453125, |
|
"logps/rejected": -633.0894775390625, |
|
"loss": 0.4482, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -2.5701141357421875, |
|
"rewards/margins": 1.3424403667449951, |
|
"rewards/rejected": -3.9125542640686035, |
|
"step": 1500 |
|
}, |
|
{ |
|
"epoch": 0.7851347814708192, |
|
"eval_logits/chosen": -0.8246029615402222, |
|
"eval_logits/rejected": -0.8422741293907166, |
|
"eval_logps/chosen": -527.0020751953125, |
|
"eval_logps/rejected": -624.8656005859375, |
|
"eval_loss": 0.4811870753765106, |
|
"eval_rewards/accuracies": 0.7760000228881836, |
|
"eval_rewards/chosen": -2.506087303161621, |
|
"eval_rewards/margins": 1.209954857826233, |
|
"eval_rewards/rejected": -3.7160420417785645, |
|
"eval_runtime": 491.111, |
|
"eval_samples_per_second": 4.072, |
|
"eval_steps_per_second": 0.255, |
|
"step": 1500 |
|
}, |
|
{ |
|
"epoch": 0.7903690133472913, |
|
"grad_norm": 3.367824088455391, |
|
"learning_rate": 6.387787646430854e-07, |
|
"logits/chosen": -0.7824467420578003, |
|
"logits/rejected": -0.8199766874313354, |
|
"logps/chosen": -542.2174682617188, |
|
"logps/rejected": -609.8660278320312, |
|
"loss": 0.4931, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -2.7443554401397705, |
|
"rewards/margins": 1.1063810586929321, |
|
"rewards/rejected": -3.850736141204834, |
|
"step": 1510 |
|
}, |
|
{ |
|
"epoch": 0.7956032452237635, |
|
"grad_norm": 2.87877789191255, |
|
"learning_rate": 6.085875250329401e-07, |
|
"logits/chosen": -0.8328324556350708, |
|
"logits/rejected": -0.8468655347824097, |
|
"logps/chosen": -507.84844970703125, |
|
"logps/rejected": -600.4588623046875, |
|
"loss": 0.4568, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -2.569871187210083, |
|
"rewards/margins": 1.0555684566497803, |
|
"rewards/rejected": -3.625439405441284, |
|
"step": 1520 |
|
}, |
|
{ |
|
"epoch": 0.8008374771002356, |
|
"grad_norm": 3.688216572569841, |
|
"learning_rate": 5.79028001824894e-07, |
|
"logits/chosen": -0.7519486546516418, |
|
"logits/rejected": -0.7730967402458191, |
|
"logps/chosen": -484.6551208496094, |
|
"logps/rejected": -676.157958984375, |
|
"loss": 0.429, |
|
"rewards/accuracies": 0.824999988079071, |
|
"rewards/chosen": -2.3991482257843018, |
|
"rewards/margins": 1.4251246452331543, |
|
"rewards/rejected": -3.824272871017456, |
|
"step": 1530 |
|
}, |
|
{ |
|
"epoch": 0.8060717089767077, |
|
"grad_norm": 2.22556494947782, |
|
"learning_rate": 5.501100676595761e-07, |
|
"logits/chosen": -0.8356014490127563, |
|
"logits/rejected": -0.8369625210762024, |
|
"logps/chosen": -538.4762573242188, |
|
"logps/rejected": -637.4407958984375, |
|
"loss": 0.462, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.500579833984375, |
|
"rewards/margins": 1.1396005153656006, |
|
"rewards/rejected": -3.6401805877685547, |
|
"step": 1540 |
|
}, |
|
{ |
|
"epoch": 0.8113059408531798, |
|
"grad_norm": 3.6363110356872355, |
|
"learning_rate": 5.218433808920884e-07, |
|
"logits/chosen": -0.8045191764831543, |
|
"logits/rejected": -0.8361748456954956, |
|
"logps/chosen": -514.1395263671875, |
|
"logps/rejected": -600.3705444335938, |
|
"loss": 0.4748, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -2.4007575511932373, |
|
"rewards/margins": 1.0883753299713135, |
|
"rewards/rejected": -3.4891326427459717, |
|
"step": 1550 |
|
}, |
|
{ |
|
"epoch": 0.816540172729652, |
|
"grad_norm": 4.618670652390617, |
|
"learning_rate": 4.942373823661928e-07, |
|
"logits/chosen": -0.8143288493156433, |
|
"logits/rejected": -0.8433948755264282, |
|
"logps/chosen": -569.4879760742188, |
|
"logps/rejected": -613.3157348632812, |
|
"loss": 0.5362, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -2.3494620323181152, |
|
"rewards/margins": 1.1645643711090088, |
|
"rewards/rejected": -3.514026641845703, |
|
"step": 1560 |
|
}, |
|
{ |
|
"epoch": 0.821774404606124, |
|
"grad_norm": 2.7457651434809893, |
|
"learning_rate": 4.6730129226114363e-07, |
|
"logits/chosen": -0.8218878507614136, |
|
"logits/rejected": -0.8484194874763489, |
|
"logps/chosen": -491.6686096191406, |
|
"logps/rejected": -611.8516845703125, |
|
"loss": 0.4799, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -2.467026710510254, |
|
"rewards/margins": 1.123768925666809, |
|
"rewards/rejected": -3.5907950401306152, |
|
"step": 1570 |
|
}, |
|
{ |
|
"epoch": 0.8270086364825961, |
|
"grad_norm": 2.68839096035273, |
|
"learning_rate": 4.4104410701222703e-07, |
|
"logits/chosen": -0.8048428297042847, |
|
"logits/rejected": -0.8185631036758423, |
|
"logps/chosen": -523.8101196289062, |
|
"logps/rejected": -619.0768432617188, |
|
"loss": 0.5228, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -2.5985608100891113, |
|
"rewards/margins": 0.9621597528457642, |
|
"rewards/rejected": -3.560720443725586, |
|
"step": 1580 |
|
}, |
|
{ |
|
"epoch": 0.8322428683590684, |
|
"grad_norm": 3.1572366237408316, |
|
"learning_rate": 4.154745963060197e-07, |
|
"logits/chosen": -0.7831433415412903, |
|
"logits/rejected": -0.7975921034812927, |
|
"logps/chosen": -479.276123046875, |
|
"logps/rejected": -577.49169921875, |
|
"loss": 0.4298, |
|
"rewards/accuracies": 0.8062499761581421, |
|
"rewards/chosen": -2.3617072105407715, |
|
"rewards/margins": 1.14790940284729, |
|
"rewards/rejected": -3.5096163749694824, |
|
"step": 1590 |
|
}, |
|
{ |
|
"epoch": 0.8374771002355405, |
|
"grad_norm": 3.2177419289192843, |
|
"learning_rate": 3.9060130015138863e-07, |
|
"logits/chosen": -0.7938388586044312, |
|
"logits/rejected": -0.8209096193313599, |
|
"logps/chosen": -501.1689453125, |
|
"logps/rejected": -594.0531005859375, |
|
"loss": 0.4982, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -2.3808839321136475, |
|
"rewards/margins": 1.0191795825958252, |
|
"rewards/rejected": -3.4000632762908936, |
|
"step": 1600 |
|
}, |
|
{ |
|
"epoch": 0.8374771002355405, |
|
"eval_logits/chosen": -0.8202841877937317, |
|
"eval_logits/rejected": -0.8376915454864502, |
|
"eval_logps/chosen": -499.3264465332031, |
|
"eval_logps/rejected": -592.1224365234375, |
|
"eval_loss": 0.4787100553512573, |
|
"eval_rewards/accuracies": 0.7770000100135803, |
|
"eval_rewards/chosen": -2.2293312549591064, |
|
"eval_rewards/margins": 1.159279465675354, |
|
"eval_rewards/rejected": -3.388610601425171, |
|
"eval_runtime": 490.4246, |
|
"eval_samples_per_second": 4.078, |
|
"eval_steps_per_second": 0.255, |
|
"step": 1600 |
|
}, |
|
{ |
|
"epoch": 0.8427113321120125, |
|
"grad_norm": 7.214430205771756, |
|
"learning_rate": 3.664325260271953e-07, |
|
"logits/chosen": -0.796721339225769, |
|
"logits/rejected": -0.8238224983215332, |
|
"logps/chosen": -540.4601440429688, |
|
"logps/rejected": -581.5528564453125, |
|
"loss": 0.4829, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -2.191901683807373, |
|
"rewards/margins": 0.9697766304016113, |
|
"rewards/rejected": -3.1616785526275635, |
|
"step": 1610 |
|
}, |
|
{ |
|
"epoch": 0.8479455639884846, |
|
"grad_norm": 2.9830959712764993, |
|
"learning_rate": 3.429763461076677e-07, |
|
"logits/chosen": -0.795876145362854, |
|
"logits/rejected": -0.8391523361206055, |
|
"logps/chosen": -538.1182861328125, |
|
"logps/rejected": -600.2818603515625, |
|
"loss": 0.4793, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -2.279409408569336, |
|
"rewards/margins": 1.0599138736724854, |
|
"rewards/rejected": -3.3393235206604004, |
|
"step": 1620 |
|
}, |
|
{ |
|
"epoch": 0.8531797958649568, |
|
"grad_norm": 2.9273391665482746, |
|
"learning_rate": 3.202405945663556e-07, |
|
"logits/chosen": -0.8152064085006714, |
|
"logits/rejected": -0.8364097476005554, |
|
"logps/chosen": -526.7260131835938, |
|
"logps/rejected": -597.470947265625, |
|
"loss": 0.4621, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -2.458247661590576, |
|
"rewards/margins": 1.0275027751922607, |
|
"rewards/rejected": -3.485750675201416, |
|
"step": 1630 |
|
}, |
|
{ |
|
"epoch": 0.8584140277414289, |
|
"grad_norm": 3.433887400095838, |
|
"learning_rate": 2.982328649595856e-07, |
|
"logits/chosen": -0.80964195728302, |
|
"logits/rejected": -0.8188964128494263, |
|
"logps/chosen": -488.6082458496094, |
|
"logps/rejected": -585.5677490234375, |
|
"loss": 0.4951, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.1840968132019043, |
|
"rewards/margins": 1.1141045093536377, |
|
"rewards/rejected": -3.2982017993927, |
|
"step": 1640 |
|
}, |
|
{ |
|
"epoch": 0.863648259617901, |
|
"grad_norm": 2.622521132377607, |
|
"learning_rate": 2.7696050769026954e-07, |
|
"logits/chosen": -0.8142071962356567, |
|
"logits/rejected": -0.8193332552909851, |
|
"logps/chosen": -515.5963745117188, |
|
"logps/rejected": -645.8536987304688, |
|
"loss": 0.4876, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -2.4191887378692627, |
|
"rewards/margins": 1.2284886837005615, |
|
"rewards/rejected": -3.6476776599884033, |
|
"step": 1650 |
|
}, |
|
{ |
|
"epoch": 0.8688824914943732, |
|
"grad_norm": 3.0785040031189537, |
|
"learning_rate": 2.564306275529341e-07, |
|
"logits/chosen": -0.8157971501350403, |
|
"logits/rejected": -0.8292611241340637, |
|
"logps/chosen": -481.8374938964844, |
|
"logps/rejected": -571.3180541992188, |
|
"loss": 0.489, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -2.333627700805664, |
|
"rewards/margins": 1.1653764247894287, |
|
"rewards/rejected": -3.4990038871765137, |
|
"step": 1660 |
|
}, |
|
{ |
|
"epoch": 0.8741167233708453, |
|
"grad_norm": 2.984496753224684, |
|
"learning_rate": 2.3665008136077332e-07, |
|
"logits/chosen": -0.8203511238098145, |
|
"logits/rejected": -0.8614446520805359, |
|
"logps/chosen": -540.2431030273438, |
|
"logps/rejected": -649.6897583007812, |
|
"loss": 0.4706, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.4780304431915283, |
|
"rewards/margins": 1.2412341833114624, |
|
"rewards/rejected": -3.719264507293701, |
|
"step": 1670 |
|
}, |
|
{ |
|
"epoch": 0.8793509552473174, |
|
"grad_norm": 2.6893878378493556, |
|
"learning_rate": 2.1762547565553293e-07, |
|
"logits/chosen": -0.7929474711418152, |
|
"logits/rejected": -0.8287181854248047, |
|
"logps/chosen": -522.1485595703125, |
|
"logps/rejected": -629.0785522460938, |
|
"loss": 0.4218, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -2.3336870670318604, |
|
"rewards/margins": 1.2596609592437744, |
|
"rewards/rejected": -3.5933480262756348, |
|
"step": 1680 |
|
}, |
|
{ |
|
"epoch": 0.8845851871237895, |
|
"grad_norm": 3.6443267203731198, |
|
"learning_rate": 1.993631645009747e-07, |
|
"logits/chosen": -0.7802733778953552, |
|
"logits/rejected": -0.8199615478515625, |
|
"logps/chosen": -495.28399658203125, |
|
"logps/rejected": -611.3101196289062, |
|
"loss": 0.4402, |
|
"rewards/accuracies": 0.831250011920929, |
|
"rewards/chosen": -2.3746984004974365, |
|
"rewards/margins": 1.337454915046692, |
|
"rewards/rejected": -3.712153196334839, |
|
"step": 1690 |
|
}, |
|
{ |
|
"epoch": 0.8898194190002617, |
|
"grad_norm": 3.031125869912656, |
|
"learning_rate": 1.818692473606748e-07, |
|
"logits/chosen": -0.7836586833000183, |
|
"logits/rejected": -0.8228501081466675, |
|
"logps/chosen": -522.3655395507812, |
|
"logps/rejected": -597.0633544921875, |
|
"loss": 0.4594, |
|
"rewards/accuracies": 0.7749999761581421, |
|
"rewards/chosen": -2.3934988975524902, |
|
"rewards/margins": 1.1604548692703247, |
|
"rewards/rejected": -3.5539536476135254, |
|
"step": 1700 |
|
}, |
|
{ |
|
"epoch": 0.8898194190002617, |
|
"eval_logits/chosen": -0.8378602266311646, |
|
"eval_logits/rejected": -0.8565782904624939, |
|
"eval_logps/chosen": -513.1796264648438, |
|
"eval_logps/rejected": -610.4910888671875, |
|
"eval_loss": 0.47899821400642395, |
|
"eval_rewards/accuracies": 0.7730000019073486, |
|
"eval_rewards/chosen": -2.3678627014160156, |
|
"eval_rewards/margins": 1.2044339179992676, |
|
"eval_rewards/rejected": -3.5722968578338623, |
|
"eval_runtime": 490.4611, |
|
"eval_samples_per_second": 4.078, |
|
"eval_steps_per_second": 0.255, |
|
"step": 1700 |
|
}, |
|
{ |
|
"epoch": 0.8950536508767338, |
|
"grad_norm": 3.891535373197336, |
|
"learning_rate": 1.6514956706084885e-07, |
|
"logits/chosen": -0.8305776715278625, |
|
"logits/rejected": -0.8483774065971375, |
|
"logps/chosen": -526.2332763671875, |
|
"logps/rejected": -616.9027709960938, |
|
"loss": 0.4435, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -2.3717777729034424, |
|
"rewards/margins": 1.235668659210205, |
|
"rewards/rejected": -3.6074466705322266, |
|
"step": 1710 |
|
}, |
|
{ |
|
"epoch": 0.9002878827532059, |
|
"grad_norm": 3.9743964527469595, |
|
"learning_rate": 1.4920970783889737e-07, |
|
"logits/chosen": -0.8023131489753723, |
|
"logits/rejected": -0.8172504305839539, |
|
"logps/chosen": -517.201416015625, |
|
"logps/rejected": -612.9605712890625, |
|
"loss": 0.4561, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.430854082107544, |
|
"rewards/margins": 1.1006124019622803, |
|
"rewards/rejected": -3.531466245651245, |
|
"step": 1720 |
|
}, |
|
{ |
|
"epoch": 0.9055221146296781, |
|
"grad_norm": 3.3367892778828034, |
|
"learning_rate": 1.340549934783164e-07, |
|
"logits/chosen": -0.793104887008667, |
|
"logits/rejected": -0.8096216320991516, |
|
"logps/chosen": -490.0877380371094, |
|
"logps/rejected": -592.3488159179688, |
|
"loss": 0.4734, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.2823646068573, |
|
"rewards/margins": 1.2109824419021606, |
|
"rewards/rejected": -3.493346691131592, |
|
"step": 1730 |
|
}, |
|
{ |
|
"epoch": 0.9107563465061502, |
|
"grad_norm": 3.7031601561414935, |
|
"learning_rate": 1.196904855305961e-07, |
|
"logits/chosen": -0.7839618921279907, |
|
"logits/rejected": -0.7986418008804321, |
|
"logps/chosen": -523.5616455078125, |
|
"logps/rejected": -599.9172973632812, |
|
"loss": 0.493, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -2.390275001525879, |
|
"rewards/margins": 1.185589075088501, |
|
"rewards/rejected": -3.57586407661438, |
|
"step": 1740 |
|
}, |
|
{ |
|
"epoch": 0.9159905783826223, |
|
"grad_norm": 3.0931062812733674, |
|
"learning_rate": 1.0612098162470302e-07, |
|
"logits/chosen": -0.8060128092765808, |
|
"logits/rejected": -0.8450511693954468, |
|
"logps/chosen": -531.2606201171875, |
|
"logps/rejected": -596.30224609375, |
|
"loss": 0.5061, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": -2.386965274810791, |
|
"rewards/margins": 1.1063708066940308, |
|
"rewards/rejected": -3.493335723876953, |
|
"step": 1750 |
|
}, |
|
{ |
|
"epoch": 0.9212248102590945, |
|
"grad_norm": 3.0522935551827777, |
|
"learning_rate": 9.335101386471285e-08, |
|
"logits/chosen": -0.8363837003707886, |
|
"logits/rejected": -0.8385022282600403, |
|
"logps/chosen": -485.2688903808594, |
|
"logps/rejected": -600.6049194335938, |
|
"loss": 0.4747, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -2.374671459197998, |
|
"rewards/margins": 1.2024142742156982, |
|
"rewards/rejected": -3.5770859718322754, |
|
"step": 1760 |
|
}, |
|
{ |
|
"epoch": 0.9264590421355666, |
|
"grad_norm": 3.816198597418785, |
|
"learning_rate": 8.138484731612273e-08, |
|
"logits/chosen": -0.7746875286102295, |
|
"logits/rejected": -0.7804522514343262, |
|
"logps/chosen": -511.98760986328125, |
|
"logps/rejected": -612.0762329101562, |
|
"loss": 0.4838, |
|
"rewards/accuracies": 0.793749988079071, |
|
"rewards/chosen": -2.5132033824920654, |
|
"rewards/margins": 1.177661657333374, |
|
"rewards/rejected": -3.6908645629882812, |
|
"step": 1770 |
|
}, |
|
{ |
|
"epoch": 0.9316932740120387, |
|
"grad_norm": 2.5117854405272495, |
|
"learning_rate": 7.022647858135501e-08, |
|
"logits/chosen": -0.8066379427909851, |
|
"logits/rejected": -0.8451001048088074, |
|
"logps/chosen": -493.4380798339844, |
|
"logps/rejected": -612.2415771484375, |
|
"loss": 0.4947, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -2.3315136432647705, |
|
"rewards/margins": 1.325115442276001, |
|
"rewards/rejected": -3.6566288471221924, |
|
"step": 1780 |
|
}, |
|
{ |
|
"epoch": 0.9369275058885108, |
|
"grad_norm": 2.293063931614578, |
|
"learning_rate": 5.987963446492384e-08, |
|
"logits/chosen": -0.7881379723548889, |
|
"logits/rejected": -0.8155984878540039, |
|
"logps/chosen": -554.5796508789062, |
|
"logps/rejected": -694.9855346679688, |
|
"loss": 0.4453, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -2.371554136276245, |
|
"rewards/margins": 1.3966000080108643, |
|
"rewards/rejected": -3.7681541442871094, |
|
"step": 1790 |
|
}, |
|
{ |
|
"epoch": 0.942161737764983, |
|
"grad_norm": 3.9098036406700367, |
|
"learning_rate": 5.034777072871394e-08, |
|
"logits/chosen": -0.7948960065841675, |
|
"logits/rejected": -0.817171573638916, |
|
"logps/chosen": -482.91650390625, |
|
"logps/rejected": -598.6817626953125, |
|
"loss": 0.4551, |
|
"rewards/accuracies": 0.7875000238418579, |
|
"rewards/chosen": -2.3254570960998535, |
|
"rewards/margins": 1.1948542594909668, |
|
"rewards/rejected": -3.5203113555908203, |
|
"step": 1800 |
|
}, |
|
{ |
|
"epoch": 0.942161737764983, |
|
"eval_logits/chosen": -0.8396913409233093, |
|
"eval_logits/rejected": -0.8587289452552795, |
|
"eval_logps/chosen": -509.1396789550781, |
|
"eval_logps/rejected": -605.8721923828125, |
|
"eval_loss": 0.4786478281021118, |
|
"eval_rewards/accuracies": 0.7730000019073486, |
|
"eval_rewards/chosen": -2.3274641036987305, |
|
"eval_rewards/margins": 1.1986435651779175, |
|
"eval_rewards/rejected": -3.5261073112487793, |
|
"eval_runtime": 490.7513, |
|
"eval_samples_per_second": 4.075, |
|
"eval_steps_per_second": 0.255, |
|
"step": 1800 |
|
}, |
|
{ |
|
"epoch": 0.9473959696414551, |
|
"grad_norm": 3.3434570748221177, |
|
"learning_rate": 4.163407093778243e-08, |
|
"logits/chosen": -0.7641734480857849, |
|
"logits/rejected": -0.7936812043190002, |
|
"logps/chosen": -508.19061279296875, |
|
"logps/rejected": -573.2919311523438, |
|
"loss": 0.4779, |
|
"rewards/accuracies": 0.762499988079071, |
|
"rewards/chosen": -2.2934184074401855, |
|
"rewards/margins": 1.2088220119476318, |
|
"rewards/rejected": -3.5022406578063965, |
|
"step": 1810 |
|
}, |
|
{ |
|
"epoch": 0.9526302015179272, |
|
"grad_norm": 3.306806286518662, |
|
"learning_rate": 3.37414453970758e-08, |
|
"logits/chosen": -0.7990435361862183, |
|
"logits/rejected": -0.8609498739242554, |
|
"logps/chosen": -522.0191650390625, |
|
"logps/rejected": -591.8367309570312, |
|
"loss": 0.4377, |
|
"rewards/accuracies": 0.8125, |
|
"rewards/chosen": -2.2719483375549316, |
|
"rewards/margins": 1.3299857378005981, |
|
"rewards/rejected": -3.601933717727661, |
|
"step": 1820 |
|
}, |
|
{ |
|
"epoch": 0.9578644333943994, |
|
"grad_norm": 3.622993124197589, |
|
"learning_rate": 2.6672530179410183e-08, |
|
"logits/chosen": -0.8073042631149292, |
|
"logits/rejected": -0.823306679725647, |
|
"logps/chosen": -486.38525390625, |
|
"logps/rejected": -573.3196411132812, |
|
"loss": 0.493, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -2.312579393386841, |
|
"rewards/margins": 1.030884027481079, |
|
"rewards/rejected": -3.34346342086792, |
|
"step": 1830 |
|
}, |
|
{ |
|
"epoch": 0.9630986652708715, |
|
"grad_norm": 3.2673896792966106, |
|
"learning_rate": 2.04296862450451e-08, |
|
"logits/chosen": -0.8151887059211731, |
|
"logits/rejected": -0.8248012661933899, |
|
"logps/chosen": -551.4527587890625, |
|
"logps/rejected": -624.1947631835938, |
|
"loss": 0.4627, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -2.5660691261291504, |
|
"rewards/margins": 1.0703225135803223, |
|
"rewards/rejected": -3.6363918781280518, |
|
"step": 1840 |
|
}, |
|
{ |
|
"epoch": 0.9683328971473436, |
|
"grad_norm": 2.95865023129791, |
|
"learning_rate": 1.501499865314171e-08, |
|
"logits/chosen": -0.7353194952011108, |
|
"logits/rejected": -0.7695177793502808, |
|
"logps/chosen": -559.2681884765625, |
|
"logps/rejected": -642.7736206054688, |
|
"loss": 0.4444, |
|
"rewards/accuracies": 0.768750011920929, |
|
"rewards/chosen": -2.3837671279907227, |
|
"rewards/margins": 1.1292965412139893, |
|
"rewards/rejected": -3.513063907623291, |
|
"step": 1850 |
|
}, |
|
{ |
|
"epoch": 0.9735671290238157, |
|
"grad_norm": 2.492804384265188, |
|
"learning_rate": 1.0430275865371265e-08, |
|
"logits/chosen": -0.799170732498169, |
|
"logits/rejected": -0.8115439414978027, |
|
"logps/chosen": -491.9395446777344, |
|
"logps/rejected": -578.8630981445312, |
|
"loss": 0.4702, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -2.4343132972717285, |
|
"rewards/margins": 1.1897588968276978, |
|
"rewards/rejected": -3.6240718364715576, |
|
"step": 1860 |
|
}, |
|
{ |
|
"epoch": 0.9788013609002879, |
|
"grad_norm": 4.425829277630147, |
|
"learning_rate": 6.677049141901315e-09, |
|
"logits/chosen": -0.7830789685249329, |
|
"logits/rejected": -0.8174635171890259, |
|
"logps/chosen": -483.9396057128906, |
|
"logps/rejected": -624.7434692382812, |
|
"loss": 0.4794, |
|
"rewards/accuracies": 0.731249988079071, |
|
"rewards/chosen": -2.34547758102417, |
|
"rewards/margins": 1.2174358367919922, |
|
"rewards/rejected": -3.562913417816162, |
|
"step": 1870 |
|
}, |
|
{ |
|
"epoch": 0.98403559277676, |
|
"grad_norm": 2.8797731827784325, |
|
"learning_rate": 3.756572029968708e-09, |
|
"logits/chosen": -0.8215857744216919, |
|
"logits/rejected": -0.8443831205368042, |
|
"logps/chosen": -508.30419921875, |
|
"logps/rejected": -605.548828125, |
|
"loss": 0.4655, |
|
"rewards/accuracies": 0.78125, |
|
"rewards/chosen": -2.2893662452697754, |
|
"rewards/margins": 1.2450931072235107, |
|
"rewards/rejected": -3.534459352493286, |
|
"step": 1880 |
|
}, |
|
{ |
|
"epoch": 0.9892698246532321, |
|
"grad_norm": 4.192074572610901, |
|
"learning_rate": 1.6698199452053199e-09, |
|
"logits/chosen": -0.7996589541435242, |
|
"logits/rejected": -0.8368595838546753, |
|
"logps/chosen": -520.9606323242188, |
|
"logps/rejected": -608.918701171875, |
|
"loss": 0.482, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -2.332554340362549, |
|
"rewards/margins": 1.064354658126831, |
|
"rewards/rejected": -3.39690899848938, |
|
"step": 1890 |
|
}, |
|
{ |
|
"epoch": 0.9945040565297043, |
|
"grad_norm": 2.981651951733524, |
|
"learning_rate": 4.1748984585560094e-10, |
|
"logits/chosen": -0.80968177318573, |
|
"logits/rejected": -0.794297993183136, |
|
"logps/chosen": -501.98553466796875, |
|
"logps/rejected": -608.4963989257812, |
|
"loss": 0.4605, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -2.3502135276794434, |
|
"rewards/margins": 1.3177697658538818, |
|
"rewards/rejected": -3.667983293533325, |
|
"step": 1900 |
|
}, |
|
{ |
|
"epoch": 0.9945040565297043, |
|
"eval_logits/chosen": -0.8360199928283691, |
|
"eval_logits/rejected": -0.8543941378593445, |
|
"eval_logps/chosen": -507.2547607421875, |
|
"eval_logps/rejected": -604.1884765625, |
|
"eval_loss": 0.47848692536354065, |
|
"eval_rewards/accuracies": 0.7739999890327454, |
|
"eval_rewards/chosen": -2.3086142539978027, |
|
"eval_rewards/margins": 1.2006564140319824, |
|
"eval_rewards/rejected": -3.509270668029785, |
|
"eval_runtime": 492.0941, |
|
"eval_samples_per_second": 4.064, |
|
"eval_steps_per_second": 0.254, |
|
"step": 1900 |
|
}, |
|
{ |
|
"epoch": 0.9997382884061764, |
|
"grad_norm": 3.6416690762006327, |
|
"learning_rate": 0.0, |
|
"logits/chosen": -0.8213506937026978, |
|
"logits/rejected": -0.8055696487426758, |
|
"logps/chosen": -526.1131591796875, |
|
"logps/rejected": -621.0297241210938, |
|
"loss": 0.4759, |
|
"rewards/accuracies": 0.7562500238418579, |
|
"rewards/chosen": -2.379620313644409, |
|
"rewards/margins": 1.0635095834732056, |
|
"rewards/rejected": -3.4431300163269043, |
|
"step": 1910 |
|
}, |
|
{ |
|
"epoch": 0.9997382884061764, |
|
"step": 1910, |
|
"total_flos": 0.0, |
|
"train_loss": 0.504705511212973, |
|
"train_runtime": 53071.3067, |
|
"train_samples_per_second": 1.152, |
|
"train_steps_per_second": 0.036 |
|
} |
|
], |
|
"logging_steps": 10, |
|
"max_steps": 1910, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 100, |
|
"stateful_callbacks": { |
|
"TrainerControl": { |
|
"args": { |
|
"should_epoch_stop": false, |
|
"should_evaluate": false, |
|
"should_log": false, |
|
"should_save": true, |
|
"should_training_stop": false |
|
}, |
|
"attributes": {} |
|
} |
|
}, |
|
"total_flos": 0.0, |
|
"train_batch_size": 4, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|