diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,23868 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 100, + "global_step": 15284, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 3.270111183780249e-09, + "logits/chosen": -2.634561777114868, + "logits/rejected": -2.673060417175293, + "logps/chosen": -207.5323944091797, + "logps/rejected": -286.9266052246094, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.0, + "learning_rate": 3.270111183780249e-08, + "logits/chosen": -2.217411756515503, + "logits/rejected": -1.964966893196106, + "logps/chosen": -185.95359802246094, + "logps/rejected": -165.4095458984375, + "loss": 0.6934, + "rewards/accuracies": 0.2222222238779068, + "rewards/chosen": -0.0003319006646052003, + "rewards/margins": -0.0005709417164325714, + "rewards/rejected": 0.00023904111003503203, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 6.540222367560497e-08, + "logits/chosen": -2.4321327209472656, + "logits/rejected": -2.22309947013855, + "logps/chosen": -232.46939086914062, + "logps/rejected": -231.3448028564453, + "loss": 0.6936, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.00021246029064059258, + "rewards/margins": -0.0009949529776349664, + "rewards/rejected": 0.0007824926869943738, + "step": 20 + }, + { + "epoch": 0.0, + "learning_rate": 9.810333551340746e-08, + "logits/chosen": -2.2580509185791016, + "logits/rejected": -2.162532091140747, + "logps/chosen": -197.32386779785156, + "logps/rejected": -219.0714111328125, + "loss": 0.6927, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.0010027495445683599, + "rewards/margins": 0.0008030139142647386, + "rewards/rejected": 0.000199735484784469, + "step": 30 + }, + { + "epoch": 0.0, + "learning_rate": 1.3080444735120995e-07, + "logits/chosen": -2.2124183177948, + "logits/rejected": -2.2520499229431152, + "logps/chosen": -276.0278015136719, + "logps/rejected": -265.83056640625, + "loss": 0.693, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.0004014628066215664, + "rewards/margins": 0.00031038964516483247, + "rewards/rejected": -0.0007118523353710771, + "step": 40 + }, + { + "epoch": 0.0, + "learning_rate": 1.6350555918901243e-07, + "logits/chosen": -2.3488545417785645, + "logits/rejected": -2.1415200233459473, + "logps/chosen": -204.8636932373047, + "logps/rejected": -184.67652893066406, + "loss": 0.6935, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.0005814622272737324, + "rewards/margins": -0.0007665859302505851, + "rewards/rejected": 0.0001851238775998354, + "step": 50 + }, + { + "epoch": 0.0, + "learning_rate": 1.9620667102681492e-07, + "logits/chosen": -2.306964635848999, + "logits/rejected": -2.0676581859588623, + "logps/chosen": -209.7549285888672, + "logps/rejected": -185.88958740234375, + "loss": 0.6934, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.000509511271957308, + "rewards/margins": -0.0004293478559702635, + "rewards/rejected": 0.0009388591279275715, + "step": 60 + }, + { + "epoch": 0.0, + "learning_rate": 2.289077828646174e-07, + "logits/chosen": -2.270014524459839, + "logits/rejected": -2.157578229904175, + "logps/chosen": -218.0652618408203, + "logps/rejected": -207.98446655273438, + "loss": 0.6932, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": 0.0004429142572917044, + "rewards/margins": -0.00011908942542504519, + "rewards/rejected": 0.0005620036390610039, + "step": 70 + }, + { + "epoch": 0.01, + "learning_rate": 2.616088947024199e-07, + "logits/chosen": -2.5093493461608887, + "logits/rejected": -2.2287230491638184, + "logps/chosen": -258.8407287597656, + "logps/rejected": -213.6261444091797, + "loss": 0.6934, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.0010970671428367496, + "rewards/margins": -0.0004738843417726457, + "rewards/rejected": 0.0015709514264017344, + "step": 80 + }, + { + "epoch": 0.01, + "learning_rate": 2.943100065402224e-07, + "logits/chosen": -2.258004665374756, + "logits/rejected": -2.173494815826416, + "logps/chosen": -184.66207885742188, + "logps/rejected": -165.5071563720703, + "loss": 0.6928, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.001227168133482337, + "rewards/margins": 0.0007357962313108146, + "rewards/rejected": 0.0004913717857562006, + "step": 90 + }, + { + "epoch": 0.01, + "learning_rate": 3.2701111837802487e-07, + "logits/chosen": -2.4309122562408447, + "logits/rejected": -2.425734758377075, + "logps/chosen": -168.77853393554688, + "logps/rejected": -183.8974609375, + "loss": 0.6933, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": 0.0015637589385733008, + "rewards/margins": -0.0003523774794302881, + "rewards/rejected": 0.0019161365926265717, + "step": 100 + }, + { + "epoch": 0.01, + "eval_logits/chosen": -2.349412441253662, + "eval_logits/rejected": -2.1609179973602295, + "eval_logps/chosen": -231.77978515625, + "eval_logps/rejected": -211.4759979248047, + "eval_loss": 0.6927058100700378, + "eval_rewards/accuracies": 0.4950000047683716, + "eval_rewards/chosen": 0.0022517528850585222, + "eval_rewards/margins": 0.0008927026065066457, + "eval_rewards/rejected": 0.0013590500457212329, + "eval_runtime": 735.1102, + "eval_samples_per_second": 2.721, + "eval_steps_per_second": 1.36, + "step": 100 + }, + { + "epoch": 0.01, + "learning_rate": 3.5971223021582736e-07, + "logits/chosen": -2.348449230194092, + "logits/rejected": -2.0000503063201904, + "logps/chosen": -222.5101776123047, + "logps/rejected": -167.06593322753906, + "loss": 0.6926, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0028537281323224306, + "rewards/margins": 0.0011668555671349168, + "rewards/rejected": 0.0016868729144334793, + "step": 110 + }, + { + "epoch": 0.01, + "learning_rate": 3.9241334205362984e-07, + "logits/chosen": -2.3394782543182373, + "logits/rejected": -2.2437832355499268, + "logps/chosen": -224.00106811523438, + "logps/rejected": -234.1225128173828, + "loss": 0.6929, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": 0.0029105721041560173, + "rewards/margins": 0.0004235516826156527, + "rewards/rejected": 0.002487020567059517, + "step": 120 + }, + { + "epoch": 0.01, + "learning_rate": 4.251144538914324e-07, + "logits/chosen": -2.2624309062957764, + "logits/rejected": -2.218439817428589, + "logps/chosen": -149.37326049804688, + "logps/rejected": -148.31546020507812, + "loss": 0.6923, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.002726006554439664, + "rewards/margins": 0.0016976374899968505, + "rewards/rejected": 0.0010283689480274916, + "step": 130 + }, + { + "epoch": 0.01, + "learning_rate": 4.578155657292348e-07, + "logits/chosen": -2.322082996368408, + "logits/rejected": -2.2224292755126953, + "logps/chosen": -225.61788940429688, + "logps/rejected": -159.54702758789062, + "loss": 0.6922, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0034105959348380566, + "rewards/margins": 0.001969530712813139, + "rewards/rejected": 0.0014410652220249176, + "step": 140 + }, + { + "epoch": 0.01, + "learning_rate": 4.905166775670374e-07, + "logits/chosen": -2.366978406906128, + "logits/rejected": -2.158670425415039, + "logps/chosen": -231.0216827392578, + "logps/rejected": -229.12841796875, + "loss": 0.6916, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.003339612390846014, + "rewards/margins": 0.0030463889706879854, + "rewards/rejected": 0.0002932233619503677, + "step": 150 + }, + { + "epoch": 0.01, + "learning_rate": 5.232177894048398e-07, + "logits/chosen": -2.2138314247131348, + "logits/rejected": -2.2267518043518066, + "logps/chosen": -260.34014892578125, + "logps/rejected": -224.89797973632812, + "loss": 0.691, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.0052833338268101215, + "rewards/margins": 0.004268099553883076, + "rewards/rejected": 0.0010152345057576895, + "step": 160 + }, + { + "epoch": 0.01, + "learning_rate": 5.559189012426422e-07, + "logits/chosen": -2.313904047012329, + "logits/rejected": -2.036227226257324, + "logps/chosen": -180.5023193359375, + "logps/rejected": -156.74676513671875, + "loss": 0.6921, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.003502651583403349, + "rewards/margins": 0.002065155887976289, + "rewards/rejected": 0.001437495811842382, + "step": 170 + }, + { + "epoch": 0.01, + "learning_rate": 5.886200130804448e-07, + "logits/chosen": -2.397557258605957, + "logits/rejected": -2.3370261192321777, + "logps/chosen": -217.43923950195312, + "logps/rejected": -198.5919647216797, + "loss": 0.6903, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.00789332203567028, + "rewards/margins": 0.005768561270087957, + "rewards/rejected": 0.002124760765582323, + "step": 180 + }, + { + "epoch": 0.01, + "learning_rate": 6.213211249182473e-07, + "logits/chosen": -2.068686008453369, + "logits/rejected": -2.1735379695892334, + "logps/chosen": -190.86375427246094, + "logps/rejected": -208.5029754638672, + "loss": 0.6908, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.006324954330921173, + "rewards/margins": 0.004828146193176508, + "rewards/rejected": 0.001496807555668056, + "step": 190 + }, + { + "epoch": 0.01, + "learning_rate": 6.540222367560497e-07, + "logits/chosen": -2.2759697437286377, + "logits/rejected": -2.2394649982452393, + "logps/chosen": -146.51278686523438, + "logps/rejected": -177.26605224609375, + "loss": 0.691, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.00699248630553484, + "rewards/margins": 0.0043672826141119, + "rewards/rejected": 0.002625203924253583, + "step": 200 + }, + { + "epoch": 0.01, + "eval_logits/chosen": -2.346784830093384, + "eval_logits/rejected": -2.158590316772461, + "eval_logps/chosen": -231.0670166015625, + "eval_logps/rejected": -211.30332946777344, + "eval_loss": 0.6900485754013062, + "eval_rewards/accuracies": 0.5824999809265137, + "eval_rewards/chosen": 0.009379291906952858, + "eval_rewards/margins": 0.0062935808673501015, + "eval_rewards/rejected": 0.003085711970925331, + "eval_runtime": 732.1877, + "eval_samples_per_second": 2.732, + "eval_steps_per_second": 1.366, + "step": 200 + }, + { + "epoch": 0.01, + "learning_rate": 6.867233485938523e-07, + "logits/chosen": -2.4248156547546387, + "logits/rejected": -2.2009010314941406, + "logps/chosen": -218.31198120117188, + "logps/rejected": -187.88546752929688, + "loss": 0.6885, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.012215192429721355, + "rewards/margins": 0.009410833939909935, + "rewards/rejected": 0.002804358024150133, + "step": 210 + }, + { + "epoch": 0.01, + "learning_rate": 7.194244604316547e-07, + "logits/chosen": -2.211219072341919, + "logits/rejected": -2.0395007133483887, + "logps/chosen": -182.6593475341797, + "logps/rejected": -174.7406768798828, + "loss": 0.6908, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": 0.006770218722522259, + "rewards/margins": 0.004744062200188637, + "rewards/rejected": 0.0020261567551642656, + "step": 220 + }, + { + "epoch": 0.02, + "learning_rate": 7.521255722694571e-07, + "logits/chosen": -2.418362855911255, + "logits/rejected": -2.0232901573181152, + "logps/chosen": -278.5413513183594, + "logps/rejected": -183.809814453125, + "loss": 0.6896, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.012379782274365425, + "rewards/margins": 0.007237741257995367, + "rewards/rejected": 0.005142040550708771, + "step": 230 + }, + { + "epoch": 0.02, + "learning_rate": 7.848266841072597e-07, + "logits/chosen": -2.197833299636841, + "logits/rejected": -2.1625285148620605, + "logps/chosen": -214.4978485107422, + "logps/rejected": -206.1327667236328, + "loss": 0.6872, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.013530202209949493, + "rewards/margins": 0.012023130431771278, + "rewards/rejected": 0.0015070696827024221, + "step": 240 + }, + { + "epoch": 0.02, + "learning_rate": 8.175277959450622e-07, + "logits/chosen": -2.164132595062256, + "logits/rejected": -2.3230721950531006, + "logps/chosen": -218.02047729492188, + "logps/rejected": -220.7309112548828, + "loss": 0.6874, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.02405674383044243, + "rewards/margins": 0.011705279350280762, + "rewards/rejected": 0.012351466342806816, + "step": 250 + }, + { + "epoch": 0.02, + "learning_rate": 8.502289077828648e-07, + "logits/chosen": -2.500403642654419, + "logits/rejected": -2.140150547027588, + "logps/chosen": -253.9658203125, + "logps/rejected": -189.07437133789062, + "loss": 0.686, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.03189177066087723, + "rewards/margins": 0.014671159908175468, + "rewards/rejected": 0.01722061261534691, + "step": 260 + }, + { + "epoch": 0.02, + "learning_rate": 8.829300196206672e-07, + "logits/chosen": -2.4172310829162598, + "logits/rejected": -2.1231706142425537, + "logps/chosen": -246.13687133789062, + "logps/rejected": -230.90869140625, + "loss": 0.6834, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.0267398152500391, + "rewards/margins": 0.020012779161334038, + "rewards/rejected": 0.006727036088705063, + "step": 270 + }, + { + "epoch": 0.02, + "learning_rate": 9.156311314584696e-07, + "logits/chosen": -2.30179500579834, + "logits/rejected": -2.1886558532714844, + "logps/chosen": -159.56088256835938, + "logps/rejected": -146.3377227783203, + "loss": 0.6871, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": 0.02627565525472164, + "rewards/margins": 0.012460910715162754, + "rewards/rejected": 0.013814744539558887, + "step": 280 + }, + { + "epoch": 0.02, + "learning_rate": 9.483322432962722e-07, + "logits/chosen": -2.547403335571289, + "logits/rejected": -2.1543164253234863, + "logps/chosen": -281.8327941894531, + "logps/rejected": -225.8657684326172, + "loss": 0.6858, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.033460211008787155, + "rewards/margins": 0.015266304835677147, + "rewards/rejected": 0.01819390431046486, + "step": 290 + }, + { + "epoch": 0.02, + "learning_rate": 9.810333551340747e-07, + "logits/chosen": -2.336268901824951, + "logits/rejected": -2.1922969818115234, + "logps/chosen": -264.8150634765625, + "logps/rejected": -238.97476196289062, + "loss": 0.6796, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.03844591975212097, + "rewards/margins": 0.02809033915400505, + "rewards/rejected": 0.01035558246076107, + "step": 300 + }, + { + "epoch": 0.02, + "eval_logits/chosen": -2.3478732109069824, + "eval_logits/rejected": -2.159766674041748, + "eval_logps/chosen": -228.3676300048828, + "eval_logps/rejected": -210.05612182617188, + "eval_loss": 0.683213472366333, + "eval_rewards/accuracies": 0.578499972820282, + "eval_rewards/chosen": 0.036373071372509, + "eval_rewards/margins": 0.02081523835659027, + "eval_rewards/rejected": 0.015557833015918732, + "eval_runtime": 734.681, + "eval_samples_per_second": 2.722, + "eval_steps_per_second": 1.361, + "step": 300 + }, + { + "epoch": 0.02, + "learning_rate": 1.0137344669718771e-06, + "logits/chosen": -2.345863103866577, + "logits/rejected": -2.3596839904785156, + "logps/chosen": -167.11477661132812, + "logps/rejected": -156.09837341308594, + "loss": 0.6902, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.023755531758069992, + "rewards/margins": 0.006341875996440649, + "rewards/rejected": 0.017413653433322906, + "step": 310 + }, + { + "epoch": 0.02, + "learning_rate": 1.0464355788096796e-06, + "logits/chosen": -2.451446056365967, + "logits/rejected": -2.0573291778564453, + "logps/chosen": -221.1422119140625, + "logps/rejected": -192.30712890625, + "loss": 0.6841, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.040228985249996185, + "rewards/margins": 0.019158074632287025, + "rewards/rejected": 0.02107091248035431, + "step": 320 + }, + { + "epoch": 0.02, + "learning_rate": 1.079136690647482e-06, + "logits/chosen": -2.431704044342041, + "logits/rejected": -2.1997568607330322, + "logps/chosen": -202.93020629882812, + "logps/rejected": -176.2235870361328, + "loss": 0.6766, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.054810572415590286, + "rewards/margins": 0.034405697137117386, + "rewards/rejected": 0.02040487341582775, + "step": 330 + }, + { + "epoch": 0.02, + "learning_rate": 1.1118378024852844e-06, + "logits/chosen": -2.1808204650878906, + "logits/rejected": -2.346876621246338, + "logps/chosen": -150.7911834716797, + "logps/rejected": -178.24876403808594, + "loss": 0.6939, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": 0.02923217974603176, + "rewards/margins": 0.00014748350076843053, + "rewards/rejected": 0.029084693640470505, + "step": 340 + }, + { + "epoch": 0.02, + "learning_rate": 1.144538914323087e-06, + "logits/chosen": -2.4173805713653564, + "logits/rejected": -1.9806461334228516, + "logps/chosen": -317.50555419921875, + "logps/rejected": -248.4846649169922, + "loss": 0.6748, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.04605143517255783, + "rewards/margins": 0.03825680539011955, + "rewards/rejected": 0.007794632576406002, + "step": 350 + }, + { + "epoch": 0.02, + "learning_rate": 1.1772400261608895e-06, + "logits/chosen": -2.4809823036193848, + "logits/rejected": -2.18863582611084, + "logps/chosen": -219.45248413085938, + "logps/rejected": -193.30197143554688, + "loss": 0.6742, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.06027550250291824, + "rewards/margins": 0.040139321237802505, + "rewards/rejected": 0.02013617567718029, + "step": 360 + }, + { + "epoch": 0.02, + "learning_rate": 1.2099411379986922e-06, + "logits/chosen": -2.163193464279175, + "logits/rejected": -2.23931884765625, + "logps/chosen": -191.78585815429688, + "logps/rejected": -206.739013671875, + "loss": 0.6833, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.05710885673761368, + "rewards/margins": 0.021854441612958908, + "rewards/rejected": 0.03525441139936447, + "step": 370 + }, + { + "epoch": 0.02, + "learning_rate": 1.2426422498364946e-06, + "logits/chosen": -2.3214056491851807, + "logits/rejected": -2.0346055030822754, + "logps/chosen": -215.81735229492188, + "logps/rejected": -163.44287109375, + "loss": 0.6685, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.058308642357587814, + "rewards/margins": 0.05202876403927803, + "rewards/rejected": 0.0062798731960356236, + "step": 380 + }, + { + "epoch": 0.03, + "learning_rate": 1.2753433616742968e-06, + "logits/chosen": -2.319272041320801, + "logits/rejected": -2.2392351627349854, + "logps/chosen": -181.83956909179688, + "logps/rejected": -245.88583374023438, + "loss": 0.6778, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.03461431339383125, + "rewards/margins": 0.0336410291492939, + "rewards/rejected": 0.0009732857579365373, + "step": 390 + }, + { + "epoch": 0.03, + "learning_rate": 1.3080444735120995e-06, + "logits/chosen": -2.477436065673828, + "logits/rejected": -2.1102676391601562, + "logps/chosen": -220.322265625, + "logps/rejected": -183.0004119873047, + "loss": 0.6558, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.05170411616563797, + "rewards/margins": 0.07928358018398285, + "rewards/rejected": -0.027579456567764282, + "step": 400 + }, + { + "epoch": 0.03, + "eval_logits/chosen": -2.3431003093719482, + "eval_logits/rejected": -2.1555795669555664, + "eval_logps/chosen": -228.5252685546875, + "eval_logps/rejected": -213.00389099121094, + "eval_loss": 0.6709386110305786, + "eval_rewards/accuracies": 0.6029999852180481, + "eval_rewards/chosen": 0.03479691967368126, + "eval_rewards/margins": 0.0487169548869133, + "eval_rewards/rejected": -0.013920034281909466, + "eval_runtime": 730.9676, + "eval_samples_per_second": 2.736, + "eval_steps_per_second": 1.368, + "step": 400 + }, + { + "epoch": 0.03, + "learning_rate": 1.3407455853499021e-06, + "logits/chosen": -2.4622159004211426, + "logits/rejected": -2.3118896484375, + "logps/chosen": -258.96173095703125, + "logps/rejected": -228.4738311767578, + "loss": 0.6662, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.0339154414832592, + "rewards/margins": 0.05908499285578728, + "rewards/rejected": -0.025169555097818375, + "step": 410 + }, + { + "epoch": 0.03, + "learning_rate": 1.3734466971877046e-06, + "logits/chosen": -2.288273811340332, + "logits/rejected": -2.189143657684326, + "logps/chosen": -178.7388153076172, + "logps/rejected": -177.989013671875, + "loss": 0.6625, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.03180820122361183, + "rewards/margins": 0.06718931347131729, + "rewards/rejected": -0.03538110852241516, + "step": 420 + }, + { + "epoch": 0.03, + "learning_rate": 1.406147809025507e-06, + "logits/chosen": -2.280384063720703, + "logits/rejected": -2.0939345359802246, + "logps/chosen": -212.7125244140625, + "logps/rejected": -187.62887573242188, + "loss": 0.6598, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.024347607046365738, + "rewards/margins": 0.07415790855884552, + "rewards/rejected": -0.04981030896306038, + "step": 430 + }, + { + "epoch": 0.03, + "learning_rate": 1.4388489208633094e-06, + "logits/chosen": -2.3787431716918945, + "logits/rejected": -2.0949296951293945, + "logps/chosen": -253.77340698242188, + "logps/rejected": -229.16744995117188, + "loss": 0.6698, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.007238357327878475, + "rewards/margins": 0.05305751413106918, + "rewards/rejected": -0.04581916332244873, + "step": 440 + }, + { + "epoch": 0.03, + "learning_rate": 1.471550032701112e-06, + "logits/chosen": -2.3988170623779297, + "logits/rejected": -2.199057102203369, + "logps/chosen": -188.98382568359375, + "logps/rejected": -200.36709594726562, + "loss": 0.6693, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.0066403718665242195, + "rewards/margins": 0.05581866577267647, + "rewards/rejected": -0.049178291112184525, + "step": 450 + }, + { + "epoch": 0.03, + "learning_rate": 1.5042511445389143e-06, + "logits/chosen": -2.1432154178619385, + "logits/rejected": -2.191903591156006, + "logps/chosen": -154.96592712402344, + "logps/rejected": -223.0820770263672, + "loss": 0.6729, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.04025929421186447, + "rewards/margins": 0.04690135270357132, + "rewards/rejected": -0.0871606320142746, + "step": 460 + }, + { + "epoch": 0.03, + "learning_rate": 1.536952256376717e-06, + "logits/chosen": -2.03416109085083, + "logits/rejected": -2.096924304962158, + "logps/chosen": -215.8236846923828, + "logps/rejected": -269.8562316894531, + "loss": 0.6576, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14893050491809845, + "rewards/margins": 0.08468537777662277, + "rewards/rejected": -0.233615905046463, + "step": 470 + }, + { + "epoch": 0.03, + "learning_rate": 1.5696533682145194e-06, + "logits/chosen": -2.428750514984131, + "logits/rejected": -2.1963045597076416, + "logps/chosen": -199.69581604003906, + "logps/rejected": -173.8529815673828, + "loss": 0.6704, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.15449048578739166, + "rewards/margins": 0.05215887352824211, + "rewards/rejected": -0.20664937794208527, + "step": 480 + }, + { + "epoch": 0.03, + "learning_rate": 1.602354480052322e-06, + "logits/chosen": -2.404479503631592, + "logits/rejected": -2.315060615539551, + "logps/chosen": -278.8482666015625, + "logps/rejected": -232.26596069335938, + "loss": 0.6554, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.12208490073680878, + "rewards/margins": 0.08369401842355728, + "rewards/rejected": -0.20577891170978546, + "step": 490 + }, + { + "epoch": 0.03, + "learning_rate": 1.6350555918901245e-06, + "logits/chosen": -2.1991093158721924, + "logits/rejected": -1.8723251819610596, + "logps/chosen": -228.2333984375, + "logps/rejected": -223.07217407226562, + "loss": 0.6509, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1016424298286438, + "rewards/margins": 0.0938231348991394, + "rewards/rejected": -0.1954655647277832, + "step": 500 + }, + { + "epoch": 0.03, + "eval_logits/chosen": -2.339709520339966, + "eval_logits/rejected": -2.152255058288574, + "eval_logps/chosen": -238.85260009765625, + "eval_logps/rejected": -228.26219177246094, + "eval_loss": 0.6524506211280823, + "eval_rewards/accuracies": 0.6060000061988831, + "eval_rewards/chosen": -0.06847651302814484, + "eval_rewards/margins": 0.09802625328302383, + "eval_rewards/rejected": -0.16650275886058807, + "eval_runtime": 732.3658, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.365, + "step": 500 + }, + { + "epoch": 0.03, + "learning_rate": 1.6677567037279269e-06, + "logits/chosen": -2.4804911613464355, + "logits/rejected": -2.132293701171875, + "logps/chosen": -297.90289306640625, + "logps/rejected": -274.1810302734375, + "loss": 0.6225, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.016392212361097336, + "rewards/margins": 0.16663143038749695, + "rewards/rejected": -0.18302366137504578, + "step": 510 + }, + { + "epoch": 0.03, + "learning_rate": 1.7004578155657295e-06, + "logits/chosen": -2.290311098098755, + "logits/rejected": -2.3183419704437256, + "logps/chosen": -209.4591064453125, + "logps/rejected": -201.08511352539062, + "loss": 0.6408, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.027397459372878075, + "rewards/margins": 0.12046666443347931, + "rewards/rejected": -0.14786411821842194, + "step": 520 + }, + { + "epoch": 0.03, + "learning_rate": 1.7331589274035318e-06, + "logits/chosen": -2.177031993865967, + "logits/rejected": -2.0574111938476562, + "logps/chosen": -192.4308319091797, + "logps/rejected": -207.4688720703125, + "loss": 0.6297, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.08504272997379303, + "rewards/margins": 0.15592262148857117, + "rewards/rejected": -0.2409653663635254, + "step": 530 + }, + { + "epoch": 0.04, + "learning_rate": 1.7658600392413344e-06, + "logits/chosen": -2.390712022781372, + "logits/rejected": -2.231996536254883, + "logps/chosen": -214.75546264648438, + "logps/rejected": -217.5111846923828, + "loss": 0.6082, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11900047212839127, + "rewards/margins": 0.20125310122966766, + "rewards/rejected": -0.3202535808086395, + "step": 540 + }, + { + "epoch": 0.04, + "learning_rate": 1.7985611510791368e-06, + "logits/chosen": -2.3818039894104004, + "logits/rejected": -1.9254570007324219, + "logps/chosen": -279.2803649902344, + "logps/rejected": -284.9852294921875, + "loss": 0.6098, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15730682015419006, + "rewards/margins": 0.22093351185321808, + "rewards/rejected": -0.37824031710624695, + "step": 550 + }, + { + "epoch": 0.04, + "learning_rate": 1.8312622629169393e-06, + "logits/chosen": -2.308039903640747, + "logits/rejected": -2.169069766998291, + "logps/chosen": -288.8381042480469, + "logps/rejected": -263.2790222167969, + "loss": 0.6716, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.25383657217025757, + "rewards/margins": 0.08961613476276398, + "rewards/rejected": -0.34345269203186035, + "step": 560 + }, + { + "epoch": 0.04, + "learning_rate": 1.8639633747547417e-06, + "logits/chosen": -2.396559000015259, + "logits/rejected": -2.2422266006469727, + "logps/chosen": -238.0849151611328, + "logps/rejected": -224.30810546875, + "loss": 0.6036, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08939802646636963, + "rewards/margins": 0.21860051155090332, + "rewards/rejected": -0.30799856781959534, + "step": 570 + }, + { + "epoch": 0.04, + "learning_rate": 1.8966644865925443e-06, + "logits/chosen": -2.2427303791046143, + "logits/rejected": -2.270328998565674, + "logps/chosen": -265.56524658203125, + "logps/rejected": -288.98272705078125, + "loss": 0.6622, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.26356470584869385, + "rewards/margins": 0.12023051083087921, + "rewards/rejected": -0.38379520177841187, + "step": 580 + }, + { + "epoch": 0.04, + "learning_rate": 1.9293655984303466e-06, + "logits/chosen": -2.6671719551086426, + "logits/rejected": -2.251417875289917, + "logps/chosen": -302.67108154296875, + "logps/rejected": -241.68118286132812, + "loss": 0.6275, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.19704832136631012, + "rewards/margins": 0.22014763951301575, + "rewards/rejected": -0.4171959459781647, + "step": 590 + }, + { + "epoch": 0.04, + "learning_rate": 1.9620667102681494e-06, + "logits/chosen": -2.4273905754089355, + "logits/rejected": -2.434626817703247, + "logps/chosen": -206.70437622070312, + "logps/rejected": -203.82342529296875, + "loss": 0.6521, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.17451904714107513, + "rewards/margins": 0.11439670622348785, + "rewards/rejected": -0.288915753364563, + "step": 600 + }, + { + "epoch": 0.04, + "eval_logits/chosen": -2.395611047744751, + "eval_logits/rejected": -2.204347610473633, + "eval_logps/chosen": -246.47792053222656, + "eval_logps/rejected": -243.2220001220703, + "eval_loss": 0.6305904388427734, + "eval_rewards/accuracies": 0.6010000109672546, + "eval_rewards/chosen": -0.14472965896129608, + "eval_rewards/margins": 0.1713714748620987, + "eval_rewards/rejected": -0.3161011338233948, + "eval_runtime": 732.687, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 600 + }, + { + "epoch": 0.04, + "learning_rate": 1.994767822105952e-06, + "logits/chosen": -2.361133098602295, + "logits/rejected": -2.2484490871429443, + "logps/chosen": -193.17393493652344, + "logps/rejected": -212.9612274169922, + "loss": 0.6562, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.17970283329486847, + "rewards/margins": 0.1335226595401764, + "rewards/rejected": -0.31322550773620605, + "step": 610 + }, + { + "epoch": 0.04, + "learning_rate": 2.0274689339437543e-06, + "logits/chosen": -2.2839503288269043, + "logits/rejected": -2.068014621734619, + "logps/chosen": -286.80120849609375, + "logps/rejected": -266.9192199707031, + "loss": 0.6457, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.08111458271741867, + "rewards/margins": 0.15472231805324554, + "rewards/rejected": -0.2358369082212448, + "step": 620 + }, + { + "epoch": 0.04, + "learning_rate": 2.0601700457815567e-06, + "logits/chosen": -2.422149658203125, + "logits/rejected": -2.0538346767425537, + "logps/chosen": -281.3658142089844, + "logps/rejected": -246.43167114257812, + "loss": 0.6705, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.22011089324951172, + "rewards/margins": 0.08579105138778687, + "rewards/rejected": -0.3059019446372986, + "step": 630 + }, + { + "epoch": 0.04, + "learning_rate": 2.092871157619359e-06, + "logits/chosen": -2.465773105621338, + "logits/rejected": -2.275972366333008, + "logps/chosen": -199.1258087158203, + "logps/rejected": -225.9133758544922, + "loss": 0.6435, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.22463631629943848, + "rewards/margins": 0.13076074421405792, + "rewards/rejected": -0.3553970456123352, + "step": 640 + }, + { + "epoch": 0.04, + "learning_rate": 2.1255722694571616e-06, + "logits/chosen": -2.4932544231414795, + "logits/rejected": -2.0854344367980957, + "logps/chosen": -312.80609130859375, + "logps/rejected": -245.77651977539062, + "loss": 0.6011, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.33911433815956116, + "rewards/margins": 0.2593802511692047, + "rewards/rejected": -0.5984946489334106, + "step": 650 + }, + { + "epoch": 0.04, + "learning_rate": 2.158273381294964e-06, + "logits/chosen": -2.4111924171447754, + "logits/rejected": -2.306203603744507, + "logps/chosen": -266.17041015625, + "logps/rejected": -277.33294677734375, + "loss": 0.6803, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.5374342799186707, + "rewards/margins": 0.05651041865348816, + "rewards/rejected": -0.5939447283744812, + "step": 660 + }, + { + "epoch": 0.04, + "learning_rate": 2.190974493132767e-06, + "logits/chosen": -2.3987817764282227, + "logits/rejected": -2.085740566253662, + "logps/chosen": -292.57562255859375, + "logps/rejected": -249.35812377929688, + "loss": 0.6427, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.5489651560783386, + "rewards/margins": 0.13039323687553406, + "rewards/rejected": -0.6793584227561951, + "step": 670 + }, + { + "epoch": 0.04, + "learning_rate": 2.223675604970569e-06, + "logits/chosen": -2.4640307426452637, + "logits/rejected": -2.2579846382141113, + "logps/chosen": -271.7763977050781, + "logps/rejected": -270.57086181640625, + "loss": 0.634, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.6104215383529663, + "rewards/margins": 0.16124722361564636, + "rewards/rejected": -0.7716687917709351, + "step": 680 + }, + { + "epoch": 0.05, + "learning_rate": 2.2563767168083718e-06, + "logits/chosen": -2.558981418609619, + "logits/rejected": -2.049004077911377, + "logps/chosen": -307.95623779296875, + "logps/rejected": -265.10205078125, + "loss": 0.6143, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5070446729660034, + "rewards/margins": 0.2017807513475418, + "rewards/rejected": -0.7088254690170288, + "step": 690 + }, + { + "epoch": 0.05, + "learning_rate": 2.289077828646174e-06, + "logits/chosen": -2.3857240676879883, + "logits/rejected": -2.15393328666687, + "logps/chosen": -295.8628845214844, + "logps/rejected": -278.58331298828125, + "loss": 0.6828, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.5680192708969116, + "rewards/margins": 0.07425983995199203, + "rewards/rejected": -0.6422791481018066, + "step": 700 + }, + { + "epoch": 0.05, + "eval_logits/chosen": -2.4134950637817383, + "eval_logits/rejected": -2.220484733581543, + "eval_logps/chosen": -279.9759521484375, + "eval_logps/rejected": -274.9947204589844, + "eval_loss": 0.6355435848236084, + "eval_rewards/accuracies": 0.5995000004768372, + "eval_rewards/chosen": -0.4797098934650421, + "eval_rewards/margins": 0.15411828458309174, + "eval_rewards/rejected": -0.6338281631469727, + "eval_runtime": 735.4698, + "eval_samples_per_second": 2.719, + "eval_steps_per_second": 1.36, + "step": 700 + }, + { + "epoch": 0.05, + "learning_rate": 2.3217789404839766e-06, + "logits/chosen": -2.271242141723633, + "logits/rejected": -2.3510873317718506, + "logps/chosen": -203.15206909179688, + "logps/rejected": -256.40960693359375, + "loss": 0.6759, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.46320945024490356, + "rewards/margins": 0.06314848363399506, + "rewards/rejected": -0.526357889175415, + "step": 710 + }, + { + "epoch": 0.05, + "learning_rate": 2.354480052321779e-06, + "logits/chosen": -2.6204721927642822, + "logits/rejected": -2.177229404449463, + "logps/chosen": -301.5008544921875, + "logps/rejected": -259.2355041503906, + "loss": 0.6273, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4311140179634094, + "rewards/margins": 0.17375989258289337, + "rewards/rejected": -0.6048739552497864, + "step": 720 + }, + { + "epoch": 0.05, + "learning_rate": 2.3871811641595815e-06, + "logits/chosen": -2.4361162185668945, + "logits/rejected": -2.244396686553955, + "logps/chosen": -279.8239440917969, + "logps/rejected": -245.77609252929688, + "loss": 0.5919, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.330314964056015, + "rewards/margins": 0.246080681681633, + "rewards/rejected": -0.5763957500457764, + "step": 730 + }, + { + "epoch": 0.05, + "learning_rate": 2.4198822759973843e-06, + "logits/chosen": -2.274948835372925, + "logits/rejected": -2.289696216583252, + "logps/chosen": -227.0866241455078, + "logps/rejected": -250.60888671875, + "loss": 0.6702, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.36480557918548584, + "rewards/margins": 0.09589744359254837, + "rewards/rejected": -0.4607029855251312, + "step": 740 + }, + { + "epoch": 0.05, + "learning_rate": 2.4525833878351864e-06, + "logits/chosen": -2.552551746368408, + "logits/rejected": -2.348984479904175, + "logps/chosen": -281.94403076171875, + "logps/rejected": -228.70938110351562, + "loss": 0.656, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2914310395717621, + "rewards/margins": 0.13045336306095123, + "rewards/rejected": -0.4218843877315521, + "step": 750 + }, + { + "epoch": 0.05, + "learning_rate": 2.4852844996729892e-06, + "logits/chosen": -2.296828031539917, + "logits/rejected": -2.2030820846557617, + "logps/chosen": -268.18157958984375, + "logps/rejected": -303.1499328613281, + "loss": 0.5807, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2905727028846741, + "rewards/margins": 0.33510199189186096, + "rewards/rejected": -0.6256746053695679, + "step": 760 + }, + { + "epoch": 0.05, + "learning_rate": 2.5179856115107916e-06, + "logits/chosen": -2.457963228225708, + "logits/rejected": -2.1134114265441895, + "logps/chosen": -301.0068359375, + "logps/rejected": -267.90350341796875, + "loss": 0.6252, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.27510613203048706, + "rewards/margins": 0.20844988524913788, + "rewards/rejected": -0.48355597257614136, + "step": 770 + }, + { + "epoch": 0.05, + "learning_rate": 2.5506867233485937e-06, + "logits/chosen": -2.5407819747924805, + "logits/rejected": -2.106503486633301, + "logps/chosen": -283.8204650878906, + "logps/rejected": -266.38104248046875, + "loss": 0.5409, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18124490976333618, + "rewards/margins": 0.400592565536499, + "rewards/rejected": -0.5818374752998352, + "step": 780 + }, + { + "epoch": 0.05, + "learning_rate": 2.5833878351863965e-06, + "logits/chosen": -2.586670398712158, + "logits/rejected": -2.5163044929504395, + "logps/chosen": -272.54180908203125, + "logps/rejected": -296.7222900390625, + "loss": 0.6022, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.20543861389160156, + "rewards/margins": 0.26800888776779175, + "rewards/rejected": -0.4734474718570709, + "step": 790 + }, + { + "epoch": 0.05, + "learning_rate": 2.616088947024199e-06, + "logits/chosen": -2.2410073280334473, + "logits/rejected": -2.4264373779296875, + "logps/chosen": -227.9165496826172, + "logps/rejected": -270.60357666015625, + "loss": 0.6578, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.31427091360092163, + "rewards/margins": 0.18160662055015564, + "rewards/rejected": -0.4958775043487549, + "step": 800 + }, + { + "epoch": 0.05, + "eval_logits/chosen": -2.4512248039245605, + "eval_logits/rejected": -2.2566721439361572, + "eval_logps/chosen": -273.8340759277344, + "eval_logps/rejected": -281.5426940917969, + "eval_loss": 0.6069934964179993, + "eval_rewards/accuracies": 0.6050000190734863, + "eval_rewards/chosen": -0.41829144954681396, + "eval_rewards/margins": 0.2810164988040924, + "eval_rewards/rejected": -0.6993079781532288, + "eval_runtime": 732.0392, + "eval_samples_per_second": 2.732, + "eval_steps_per_second": 1.366, + "step": 800 + }, + { + "epoch": 0.05, + "learning_rate": 2.6487900588620014e-06, + "logits/chosen": -2.2319188117980957, + "logits/rejected": -1.9784733057022095, + "logps/chosen": -248.84750366210938, + "logps/rejected": -223.2877960205078, + "loss": 0.6512, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.4873189926147461, + "rewards/margins": 0.16692177951335907, + "rewards/rejected": -0.6542407870292664, + "step": 810 + }, + { + "epoch": 0.05, + "learning_rate": 2.6814911706998042e-06, + "logits/chosen": -2.381350040435791, + "logits/rejected": -2.1919376850128174, + "logps/chosen": -273.8369140625, + "logps/rejected": -258.0627136230469, + "loss": 0.6554, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.4849766194820404, + "rewards/margins": 0.14979836344718933, + "rewards/rejected": -0.6347750425338745, + "step": 820 + }, + { + "epoch": 0.05, + "learning_rate": 2.7141922825376067e-06, + "logits/chosen": -2.2474639415740967, + "logits/rejected": -2.2449734210968018, + "logps/chosen": -300.6055908203125, + "logps/rejected": -345.5708923339844, + "loss": 0.5734, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.42681440711021423, + "rewards/margins": 0.35034364461898804, + "rewards/rejected": -0.7771580219268799, + "step": 830 + }, + { + "epoch": 0.05, + "learning_rate": 2.746893394375409e-06, + "logits/chosen": -2.365583896636963, + "logits/rejected": -2.2663304805755615, + "logps/chosen": -271.7221984863281, + "logps/rejected": -299.95050048828125, + "loss": 0.5875, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.37327784299850464, + "rewards/margins": 0.30009618401527405, + "rewards/rejected": -0.6733739972114563, + "step": 840 + }, + { + "epoch": 0.06, + "learning_rate": 2.779594506213211e-06, + "logits/chosen": -2.282376766204834, + "logits/rejected": -2.0392627716064453, + "logps/chosen": -245.7705535888672, + "logps/rejected": -261.686767578125, + "loss": 0.6677, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.5591166615486145, + "rewards/margins": 0.1622948795557022, + "rewards/rejected": -0.7214115262031555, + "step": 850 + }, + { + "epoch": 0.06, + "learning_rate": 2.812295618051014e-06, + "logits/chosen": -2.442470073699951, + "logits/rejected": -2.2847018241882324, + "logps/chosen": -313.7275695800781, + "logps/rejected": -283.5700378417969, + "loss": 0.5871, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.32766368985176086, + "rewards/margins": 0.31569644808769226, + "rewards/rejected": -0.6433601975440979, + "step": 860 + }, + { + "epoch": 0.06, + "learning_rate": 2.8449967298888164e-06, + "logits/chosen": -2.3573145866394043, + "logits/rejected": -2.129387855529785, + "logps/chosen": -213.77999877929688, + "logps/rejected": -211.1344757080078, + "loss": 0.597, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3695598244667053, + "rewards/margins": 0.2541285753250122, + "rewards/rejected": -0.6236883401870728, + "step": 870 + }, + { + "epoch": 0.06, + "learning_rate": 2.877697841726619e-06, + "logits/chosen": -2.4069507122039795, + "logits/rejected": -2.30299711227417, + "logps/chosen": -253.3367462158203, + "logps/rejected": -255.14401245117188, + "loss": 0.622, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3450094759464264, + "rewards/margins": 0.1862207055091858, + "rewards/rejected": -0.531230092048645, + "step": 880 + }, + { + "epoch": 0.06, + "learning_rate": 2.9103989535644217e-06, + "logits/chosen": -2.2997517585754395, + "logits/rejected": -2.4225234985351562, + "logps/chosen": -243.65475463867188, + "logps/rejected": -294.9205017089844, + "loss": 0.6317, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.28171128034591675, + "rewards/margins": 0.20735308527946472, + "rewards/rejected": -0.48906439542770386, + "step": 890 + }, + { + "epoch": 0.06, + "learning_rate": 2.943100065402224e-06, + "logits/chosen": -2.344895362854004, + "logits/rejected": -2.197391986846924, + "logps/chosen": -329.97265625, + "logps/rejected": -335.56512451171875, + "loss": 0.6272, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3732944428920746, + "rewards/margins": 0.20909173786640167, + "rewards/rejected": -0.5823861956596375, + "step": 900 + }, + { + "epoch": 0.06, + "eval_logits/chosen": -2.318371534347534, + "eval_logits/rejected": -2.133222818374634, + "eval_logps/chosen": -259.9873962402344, + "eval_logps/rejected": -263.5772399902344, + "eval_loss": 0.6149184107780457, + "eval_rewards/accuracies": 0.6060000061988831, + "eval_rewards/chosen": -0.27982431650161743, + "eval_rewards/margins": 0.23982904851436615, + "eval_rewards/rejected": -0.5196533799171448, + "eval_runtime": 732.3751, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.365, + "step": 900 + }, + { + "epoch": 0.06, + "learning_rate": 2.9758011772400266e-06, + "logits/chosen": -2.2377166748046875, + "logits/rejected": -2.2609286308288574, + "logps/chosen": -286.5142517089844, + "logps/rejected": -315.3929443359375, + "loss": 0.6139, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.29205089807510376, + "rewards/margins": 0.21504080295562744, + "rewards/rejected": -0.5070916414260864, + "step": 910 + }, + { + "epoch": 0.06, + "learning_rate": 3.0085022890778286e-06, + "logits/chosen": -2.2762789726257324, + "logits/rejected": -2.0410525798797607, + "logps/chosen": -217.5778350830078, + "logps/rejected": -217.69351196289062, + "loss": 0.6061, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.41838282346725464, + "rewards/margins": 0.27039122581481934, + "rewards/rejected": -0.6887739896774292, + "step": 920 + }, + { + "epoch": 0.06, + "learning_rate": 3.0412034009156314e-06, + "logits/chosen": -2.2033088207244873, + "logits/rejected": -2.3684961795806885, + "logps/chosen": -298.0721740722656, + "logps/rejected": -318.66436767578125, + "loss": 0.6334, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.6238594055175781, + "rewards/margins": 0.2485378533601761, + "rewards/rejected": -0.872397243976593, + "step": 930 + }, + { + "epoch": 0.06, + "learning_rate": 3.073904512753434e-06, + "logits/chosen": -2.3155341148376465, + "logits/rejected": -1.962912917137146, + "logps/chosen": -297.7289123535156, + "logps/rejected": -312.3240661621094, + "loss": 0.5093, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.680803120136261, + "rewards/margins": 0.49523410201072693, + "rewards/rejected": -1.1760371923446655, + "step": 940 + }, + { + "epoch": 0.06, + "learning_rate": 3.1066056245912363e-06, + "logits/chosen": -2.161400556564331, + "logits/rejected": -2.205284595489502, + "logps/chosen": -340.3791809082031, + "logps/rejected": -353.7043762207031, + "loss": 0.5633, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.0939515829086304, + "rewards/margins": 0.4528425335884094, + "rewards/rejected": -1.546794056892395, + "step": 950 + }, + { + "epoch": 0.06, + "learning_rate": 3.1393067364290387e-06, + "logits/chosen": -2.1303658485412598, + "logits/rejected": -1.9281947612762451, + "logps/chosen": -333.31866455078125, + "logps/rejected": -334.27972412109375, + "loss": 0.6545, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.0819190740585327, + "rewards/margins": 0.39546647667884827, + "rewards/rejected": -1.4773855209350586, + "step": 960 + }, + { + "epoch": 0.06, + "learning_rate": 3.1720078482668416e-06, + "logits/chosen": -2.1484124660491943, + "logits/rejected": -1.9389692544937134, + "logps/chosen": -317.99359130859375, + "logps/rejected": -306.39434814453125, + "loss": 0.5974, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.0642293691635132, + "rewards/margins": 0.35507652163505554, + "rewards/rejected": -1.4193060398101807, + "step": 970 + }, + { + "epoch": 0.06, + "learning_rate": 3.204708960104644e-06, + "logits/chosen": -2.2778122425079346, + "logits/rejected": -1.9579265117645264, + "logps/chosen": -306.3919372558594, + "logps/rejected": -315.81353759765625, + "loss": 0.5629, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.8898921012878418, + "rewards/margins": 0.4200926423072815, + "rewards/rejected": -1.3099846839904785, + "step": 980 + }, + { + "epoch": 0.06, + "learning_rate": 3.237410071942446e-06, + "logits/chosen": -2.2833445072174072, + "logits/rejected": -2.02951979637146, + "logps/chosen": -295.0680236816406, + "logps/rejected": -271.5545349121094, + "loss": 0.6827, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.8870692253112793, + "rewards/margins": 0.10050360858440399, + "rewards/rejected": -0.9875728487968445, + "step": 990 + }, + { + "epoch": 0.07, + "learning_rate": 3.270111183780249e-06, + "logits/chosen": -2.214215040206909, + "logits/rejected": -2.0094611644744873, + "logps/chosen": -289.36053466796875, + "logps/rejected": -294.6500549316406, + "loss": 0.6772, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.8933695554733276, + "rewards/margins": 0.2329256534576416, + "rewards/rejected": -1.1262950897216797, + "step": 1000 + }, + { + "epoch": 0.07, + "eval_logits/chosen": -2.273111343383789, + "eval_logits/rejected": -2.0915205478668213, + "eval_logps/chosen": -291.50830078125, + "eval_logps/rejected": -301.5699157714844, + "eval_loss": 0.5978599786758423, + "eval_rewards/accuracies": 0.612500011920929, + "eval_rewards/chosen": -0.5950338244438171, + "eval_rewards/margins": 0.30454614758491516, + "eval_rewards/rejected": -0.8995799422264099, + "eval_runtime": 730.9914, + "eval_samples_per_second": 2.736, + "eval_steps_per_second": 1.368, + "step": 1000 + }, + { + "epoch": 0.07, + "learning_rate": 3.3028122956180513e-06, + "logits/chosen": -2.149437427520752, + "logits/rejected": -2.1762747764587402, + "logps/chosen": -259.1993103027344, + "logps/rejected": -311.66009521484375, + "loss": 0.6148, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.47101396322250366, + "rewards/margins": 0.24991849064826965, + "rewards/rejected": -0.7209324836730957, + "step": 1010 + }, + { + "epoch": 0.07, + "learning_rate": 3.3355134074558538e-06, + "logits/chosen": -2.1731173992156982, + "logits/rejected": -2.0621697902679443, + "logps/chosen": -268.55120849609375, + "logps/rejected": -267.37139892578125, + "loss": 0.6092, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.4781845211982727, + "rewards/margins": 0.24407725036144257, + "rewards/rejected": -0.7222617268562317, + "step": 1020 + }, + { + "epoch": 0.07, + "learning_rate": 3.368214519293656e-06, + "logits/chosen": -2.1015899181365967, + "logits/rejected": -1.9048049449920654, + "logps/chosen": -242.2707061767578, + "logps/rejected": -249.1923828125, + "loss": 0.589, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.43300414085388184, + "rewards/margins": 0.27820152044296265, + "rewards/rejected": -0.7112056612968445, + "step": 1030 + }, + { + "epoch": 0.07, + "learning_rate": 3.400915631131459e-06, + "logits/chosen": -2.0095393657684326, + "logits/rejected": -2.0408594608306885, + "logps/chosen": -254.6264190673828, + "logps/rejected": -327.074951171875, + "loss": 0.5662, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5665396451950073, + "rewards/margins": 0.43797236680984497, + "rewards/rejected": -1.0045119524002075, + "step": 1040 + }, + { + "epoch": 0.07, + "learning_rate": 3.4336167429692615e-06, + "logits/chosen": -2.2427849769592285, + "logits/rejected": -2.028717517852783, + "logps/chosen": -272.2727966308594, + "logps/rejected": -288.40185546875, + "loss": 0.589, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.5772241353988647, + "rewards/margins": 0.32382819056510925, + "rewards/rejected": -0.9010523557662964, + "step": 1050 + }, + { + "epoch": 0.07, + "learning_rate": 3.4663178548070635e-06, + "logits/chosen": -2.0555243492126465, + "logits/rejected": -2.1193196773529053, + "logps/chosen": -274.7303771972656, + "logps/rejected": -302.765869140625, + "loss": 0.5504, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.5977641344070435, + "rewards/margins": 0.44477134943008423, + "rewards/rejected": -1.0425355434417725, + "step": 1060 + }, + { + "epoch": 0.07, + "learning_rate": 3.499018966644866e-06, + "logits/chosen": -2.1316325664520264, + "logits/rejected": -1.962110161781311, + "logps/chosen": -294.1905517578125, + "logps/rejected": -319.0716552734375, + "loss": 0.618, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.0236551761627197, + "rewards/margins": 0.3169350028038025, + "rewards/rejected": -1.3405901193618774, + "step": 1070 + }, + { + "epoch": 0.07, + "learning_rate": 3.531720078482669e-06, + "logits/chosen": -2.026108980178833, + "logits/rejected": -1.7189273834228516, + "logps/chosen": -338.9638671875, + "logps/rejected": -356.0903015136719, + "loss": 0.6328, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.130941390991211, + "rewards/margins": 0.2835771441459656, + "rewards/rejected": -1.4145182371139526, + "step": 1080 + }, + { + "epoch": 0.07, + "learning_rate": 3.5644211903204712e-06, + "logits/chosen": -2.0485777854919434, + "logits/rejected": -2.090273380279541, + "logps/chosen": -281.7182312011719, + "logps/rejected": -323.7969665527344, + "loss": 0.5626, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.8283576965332031, + "rewards/margins": 0.39265236258506775, + "rewards/rejected": -1.2210099697113037, + "step": 1090 + }, + { + "epoch": 0.07, + "learning_rate": 3.5971223021582737e-06, + "logits/chosen": -2.1952884197235107, + "logits/rejected": -2.0548617839813232, + "logps/chosen": -351.4318542480469, + "logps/rejected": -323.7904968261719, + "loss": 0.629, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.9563105702400208, + "rewards/margins": 0.2947392463684082, + "rewards/rejected": -1.2510497570037842, + "step": 1100 + }, + { + "epoch": 0.07, + "eval_logits/chosen": -2.064201831817627, + "eval_logits/rejected": -1.8958628177642822, + "eval_logps/chosen": -348.63909912109375, + "eval_logps/rejected": -370.0741882324219, + "eval_loss": 0.5842385292053223, + "eval_rewards/accuracies": 0.6255000233650208, + "eval_rewards/chosen": -1.1663414239883423, + "eval_rewards/margins": 0.4182808995246887, + "eval_rewards/rejected": -1.5846223831176758, + "eval_runtime": 731.6823, + "eval_samples_per_second": 2.733, + "eval_steps_per_second": 1.367, + "step": 1100 + }, + { + "epoch": 0.07, + "learning_rate": 3.6298234139960765e-06, + "logits/chosen": -2.1259961128234863, + "logits/rejected": -1.803380012512207, + "logps/chosen": -338.1694641113281, + "logps/rejected": -317.0995788574219, + "loss": 0.6346, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.2040038108825684, + "rewards/margins": 0.28601497411727905, + "rewards/rejected": -1.4900187253952026, + "step": 1110 + }, + { + "epoch": 0.07, + "learning_rate": 3.6625245258338785e-06, + "logits/chosen": -1.8661701679229736, + "logits/rejected": -1.800474762916565, + "logps/chosen": -327.68890380859375, + "logps/rejected": -447.7711486816406, + "loss": 0.5602, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.889104962348938, + "rewards/margins": 0.43621620535850525, + "rewards/rejected": -1.3253211975097656, + "step": 1120 + }, + { + "epoch": 0.07, + "learning_rate": 3.695225637671681e-06, + "logits/chosen": -2.0691475868225098, + "logits/rejected": -1.8956810235977173, + "logps/chosen": -290.79425048828125, + "logps/rejected": -287.4905700683594, + "loss": 0.5978, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.8351759910583496, + "rewards/margins": 0.2977235019207001, + "rewards/rejected": -1.132899522781372, + "step": 1130 + }, + { + "epoch": 0.07, + "learning_rate": 3.7279267495094834e-06, + "logits/chosen": -2.0039424896240234, + "logits/rejected": -1.8878767490386963, + "logps/chosen": -243.4080810546875, + "logps/rejected": -295.2784118652344, + "loss": 0.552, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.829811692237854, + "rewards/margins": 0.4174509048461914, + "rewards/rejected": -1.2472625970840454, + "step": 1140 + }, + { + "epoch": 0.08, + "learning_rate": 3.7606278613472863e-06, + "logits/chosen": -2.032029628753662, + "logits/rejected": -1.7188432216644287, + "logps/chosen": -389.14288330078125, + "logps/rejected": -346.46759033203125, + "loss": 0.6473, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.0687159299850464, + "rewards/margins": 0.3489280641078949, + "rewards/rejected": -1.4176441431045532, + "step": 1150 + }, + { + "epoch": 0.08, + "learning_rate": 3.7933289731850887e-06, + "logits/chosen": -1.8771930932998657, + "logits/rejected": -1.678802490234375, + "logps/chosen": -305.0873718261719, + "logps/rejected": -325.5848693847656, + "loss": 0.5391, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.7652674913406372, + "rewards/margins": 0.48755812644958496, + "rewards/rejected": -1.2528256177902222, + "step": 1160 + }, + { + "epoch": 0.08, + "learning_rate": 3.826030085022891e-06, + "logits/chosen": -2.0287744998931885, + "logits/rejected": -1.8205569982528687, + "logps/chosen": -321.78076171875, + "logps/rejected": -327.90667724609375, + "loss": 0.5623, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.7413099408149719, + "rewards/margins": 0.42277926206588745, + "rewards/rejected": -1.1640890836715698, + "step": 1170 + }, + { + "epoch": 0.08, + "learning_rate": 3.858731196860693e-06, + "logits/chosen": -2.0594873428344727, + "logits/rejected": -1.65718674659729, + "logps/chosen": -323.6650695800781, + "logps/rejected": -304.764892578125, + "loss": 0.6431, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.7283161878585815, + "rewards/margins": 0.23335444927215576, + "rewards/rejected": -0.9616705179214478, + "step": 1180 + }, + { + "epoch": 0.08, + "learning_rate": 3.891432308698496e-06, + "logits/chosen": -1.7721526622772217, + "logits/rejected": -1.7358468770980835, + "logps/chosen": -270.286865234375, + "logps/rejected": -338.5904235839844, + "loss": 0.5986, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.7505649328231812, + "rewards/margins": 0.4305826723575592, + "rewards/rejected": -1.1811474561691284, + "step": 1190 + }, + { + "epoch": 0.08, + "learning_rate": 3.924133420536299e-06, + "logits/chosen": -1.846959114074707, + "logits/rejected": -1.6761360168457031, + "logps/chosen": -326.3203125, + "logps/rejected": -327.2926940917969, + "loss": 0.6763, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.4417558908462524, + "rewards/margins": 0.3437504768371582, + "rewards/rejected": -1.785506248474121, + "step": 1200 + }, + { + "epoch": 0.08, + "eval_logits/chosen": -1.9452943801879883, + "eval_logits/rejected": -1.778209924697876, + "eval_logps/chosen": -354.6231384277344, + "eval_logps/rejected": -379.3278503417969, + "eval_loss": 0.5800026059150696, + "eval_rewards/accuracies": 0.625, + "eval_rewards/chosen": -1.2261821031570435, + "eval_rewards/margins": 0.45097729563713074, + "eval_rewards/rejected": -1.6771594285964966, + "eval_runtime": 732.3829, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.365, + "step": 1200 + }, + { + "epoch": 0.08, + "learning_rate": 3.956834532374101e-06, + "logits/chosen": -2.0891950130462646, + "logits/rejected": -1.746002197265625, + "logps/chosen": -308.41717529296875, + "logps/rejected": -328.99591064453125, + "loss": 0.4712, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.0315450429916382, + "rewards/margins": 0.6453005075454712, + "rewards/rejected": -1.6768455505371094, + "step": 1210 + }, + { + "epoch": 0.08, + "learning_rate": 3.989535644211904e-06, + "logits/chosen": -1.9306423664093018, + "logits/rejected": -1.703242301940918, + "logps/chosen": -316.7760009765625, + "logps/rejected": -343.18414306640625, + "loss": 0.5702, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.9964868426322937, + "rewards/margins": 0.4902920722961426, + "rewards/rejected": -1.486778974533081, + "step": 1220 + }, + { + "epoch": 0.08, + "learning_rate": 4.022236756049706e-06, + "logits/chosen": -2.1357779502868652, + "logits/rejected": -1.7851722240447998, + "logps/chosen": -356.35040283203125, + "logps/rejected": -368.52886962890625, + "loss": 0.5604, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7921323776245117, + "rewards/margins": 0.44653981924057007, + "rewards/rejected": -1.238672137260437, + "step": 1230 + }, + { + "epoch": 0.08, + "learning_rate": 4.054937867887509e-06, + "logits/chosen": -1.9934505224227905, + "logits/rejected": -1.5923570394515991, + "logps/chosen": -299.227783203125, + "logps/rejected": -345.69647216796875, + "loss": 0.5117, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.7496100664138794, + "rewards/margins": 0.57257080078125, + "rewards/rejected": -1.3221808671951294, + "step": 1240 + }, + { + "epoch": 0.08, + "learning_rate": 4.087638979725311e-06, + "logits/chosen": -1.8767526149749756, + "logits/rejected": -1.9476865530014038, + "logps/chosen": -310.9009704589844, + "logps/rejected": -339.83123779296875, + "loss": 0.6662, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.830611526966095, + "rewards/margins": 0.27475300431251526, + "rewards/rejected": -1.1053645610809326, + "step": 1250 + }, + { + "epoch": 0.08, + "learning_rate": 4.1203400915631135e-06, + "logits/chosen": -1.716407060623169, + "logits/rejected": -1.6570746898651123, + "logps/chosen": -339.90679931640625, + "logps/rejected": -354.4397277832031, + "loss": 0.5875, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.0257585048675537, + "rewards/margins": 0.42813658714294434, + "rewards/rejected": -1.453895092010498, + "step": 1260 + }, + { + "epoch": 0.08, + "learning_rate": 4.153041203400916e-06, + "logits/chosen": -1.982994794845581, + "logits/rejected": -1.8656692504882812, + "logps/chosen": -388.2208251953125, + "logps/rejected": -401.22601318359375, + "loss": 0.6485, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2379850149154663, + "rewards/margins": 0.37639838457107544, + "rewards/rejected": -1.614383339881897, + "step": 1270 + }, + { + "epoch": 0.08, + "learning_rate": 4.185742315238718e-06, + "logits/chosen": -1.8478248119354248, + "logits/rejected": -1.6495792865753174, + "logps/chosen": -339.2015075683594, + "logps/rejected": -353.11297607421875, + "loss": 0.6318, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.4109991788864136, + "rewards/margins": 0.25778403878211975, + "rewards/rejected": -1.668783187866211, + "step": 1280 + }, + { + "epoch": 0.08, + "learning_rate": 4.218443427076521e-06, + "logits/chosen": -1.9099308252334595, + "logits/rejected": -1.7717653512954712, + "logps/chosen": -330.18585205078125, + "logps/rejected": -346.7439880371094, + "loss": 0.6937, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.4742728471755981, + "rewards/margins": 0.15087108314037323, + "rewards/rejected": -1.6251437664031982, + "step": 1290 + }, + { + "epoch": 0.09, + "learning_rate": 4.251144538914323e-06, + "logits/chosen": -1.8934119939804077, + "logits/rejected": -1.759953498840332, + "logps/chosen": -387.2998962402344, + "logps/rejected": -393.1944274902344, + "loss": 0.6468, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.6670963764190674, + "rewards/margins": 0.19090215861797333, + "rewards/rejected": -1.8579986095428467, + "step": 1300 + }, + { + "epoch": 0.09, + "eval_logits/chosen": -2.003377676010132, + "eval_logits/rejected": -1.8354687690734863, + "eval_logps/chosen": -375.2355651855469, + "eval_logps/rejected": -384.9623718261719, + "eval_loss": 0.5959275960922241, + "eval_rewards/accuracies": 0.6265000104904175, + "eval_rewards/chosen": -1.4323062896728516, + "eval_rewards/margins": 0.3011987507343292, + "eval_rewards/rejected": -1.7335046529769897, + "eval_runtime": 733.6396, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 1300 + }, + { + "epoch": 0.09, + "learning_rate": 4.283845650752126e-06, + "logits/chosen": -2.0106637477874756, + "logits/rejected": -1.8190252780914307, + "logps/chosen": -429.07745361328125, + "logps/rejected": -397.7805480957031, + "loss": 0.5875, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.428654670715332, + "rewards/margins": 0.3289453685283661, + "rewards/rejected": -1.7576000690460205, + "step": 1310 + }, + { + "epoch": 0.09, + "learning_rate": 4.316546762589928e-06, + "logits/chosen": -1.9939743280410767, + "logits/rejected": -1.9128013849258423, + "logps/chosen": -340.85955810546875, + "logps/rejected": -335.62188720703125, + "loss": 0.5983, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3293203115463257, + "rewards/margins": 0.29060477018356323, + "rewards/rejected": -1.6199251413345337, + "step": 1320 + }, + { + "epoch": 0.09, + "learning_rate": 4.349247874427731e-06, + "logits/chosen": -1.9885427951812744, + "logits/rejected": -1.6930125951766968, + "logps/chosen": -343.95068359375, + "logps/rejected": -436.5520935058594, + "loss": 0.5516, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.1741664409637451, + "rewards/margins": 0.5251883268356323, + "rewards/rejected": -1.6993547677993774, + "step": 1330 + }, + { + "epoch": 0.09, + "learning_rate": 4.381948986265534e-06, + "logits/chosen": -2.123256206512451, + "logits/rejected": -1.8905670642852783, + "logps/chosen": -385.8031921386719, + "logps/rejected": -422.24237060546875, + "loss": 0.5249, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.2295849323272705, + "rewards/margins": 0.49446210265159607, + "rewards/rejected": -1.724047303199768, + "step": 1340 + }, + { + "epoch": 0.09, + "learning_rate": 4.414650098103336e-06, + "logits/chosen": -2.1973934173583984, + "logits/rejected": -1.9671694040298462, + "logps/chosen": -410.22314453125, + "logps/rejected": -429.04931640625, + "loss": 0.593, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.482706069946289, + "rewards/margins": 0.4125031530857086, + "rewards/rejected": -1.8952090740203857, + "step": 1350 + }, + { + "epoch": 0.09, + "learning_rate": 4.447351209941138e-06, + "logits/chosen": -1.8963654041290283, + "logits/rejected": -1.6682153940200806, + "logps/chosen": -404.86920166015625, + "logps/rejected": -426.9701232910156, + "loss": 0.5797, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.9520457983016968, + "rewards/margins": 0.40242427587509155, + "rewards/rejected": -2.3544700145721436, + "step": 1360 + }, + { + "epoch": 0.09, + "learning_rate": 4.480052321778941e-06, + "logits/chosen": -1.9020938873291016, + "logits/rejected": -1.937169075012207, + "logps/chosen": -372.9541320800781, + "logps/rejected": -379.32916259765625, + "loss": 0.6732, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.4604378938674927, + "rewards/margins": 0.27071020007133484, + "rewards/rejected": -1.73114812374115, + "step": 1370 + }, + { + "epoch": 0.09, + "learning_rate": 4.5127534336167435e-06, + "logits/chosen": -2.0027899742126465, + "logits/rejected": -1.7514774799346924, + "logps/chosen": -382.690673828125, + "logps/rejected": -369.9425048828125, + "loss": 0.6172, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3551666736602783, + "rewards/margins": 0.309119313955307, + "rewards/rejected": -1.6642862558364868, + "step": 1380 + }, + { + "epoch": 0.09, + "learning_rate": 4.5454545454545455e-06, + "logits/chosen": -1.9697223901748657, + "logits/rejected": -1.9489448070526123, + "logps/chosen": -309.4676818847656, + "logps/rejected": -308.8170471191406, + "loss": 0.6996, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.4000898599624634, + "rewards/margins": 0.09558413177728653, + "rewards/rejected": -1.4956738948822021, + "step": 1390 + }, + { + "epoch": 0.09, + "learning_rate": 4.578155657292348e-06, + "logits/chosen": -2.0561442375183105, + "logits/rejected": -2.0292887687683105, + "logps/chosen": -351.74664306640625, + "logps/rejected": -418.7784118652344, + "loss": 0.5302, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.9634095430374146, + "rewards/margins": 0.5810994505882263, + "rewards/rejected": -1.544508934020996, + "step": 1400 + }, + { + "epoch": 0.09, + "eval_logits/chosen": -2.039621353149414, + "eval_logits/rejected": -1.8706350326538086, + "eval_logps/chosen": -334.2268371582031, + "eval_logps/rejected": -353.9125671386719, + "eval_loss": 0.5790063738822937, + "eval_rewards/accuracies": 0.6370000243186951, + "eval_rewards/chosen": -1.0222188234329224, + "eval_rewards/margins": 0.4007880985736847, + "eval_rewards/rejected": -1.4230070114135742, + "eval_runtime": 733.4141, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.363, + "step": 1400 + }, + { + "epoch": 0.09, + "learning_rate": 4.610856769130151e-06, + "logits/chosen": -2.1047465801239014, + "logits/rejected": -1.9882681369781494, + "logps/chosen": -331.709228515625, + "logps/rejected": -346.4400939941406, + "loss": 0.5547, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.9256499409675598, + "rewards/margins": 0.44232434034347534, + "rewards/rejected": -1.3679742813110352, + "step": 1410 + }, + { + "epoch": 0.09, + "learning_rate": 4.643557880967953e-06, + "logits/chosen": -2.0810694694519043, + "logits/rejected": -1.894683837890625, + "logps/chosen": -282.4119873046875, + "logps/rejected": -318.69873046875, + "loss": 0.5953, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.8361316919326782, + "rewards/margins": 0.38786545395851135, + "rewards/rejected": -1.2239971160888672, + "step": 1420 + }, + { + "epoch": 0.09, + "learning_rate": 4.676258992805755e-06, + "logits/chosen": -2.0333473682403564, + "logits/rejected": -1.7796134948730469, + "logps/chosen": -363.6483459472656, + "logps/rejected": -386.46905517578125, + "loss": 0.5182, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.9402838945388794, + "rewards/margins": 0.6496909260749817, + "rewards/rejected": -1.5899746417999268, + "step": 1430 + }, + { + "epoch": 0.09, + "learning_rate": 4.708960104643558e-06, + "logits/chosen": -1.9459807872772217, + "logits/rejected": -1.925270438194275, + "logps/chosen": -426.68353271484375, + "logps/rejected": -433.5457458496094, + "loss": 0.6221, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2047381401062012, + "rewards/margins": 0.4362231194972992, + "rewards/rejected": -1.6409610509872437, + "step": 1440 + }, + { + "epoch": 0.09, + "learning_rate": 4.741661216481361e-06, + "logits/chosen": -1.8569351434707642, + "logits/rejected": -1.829811453819275, + "logps/chosen": -342.13995361328125, + "logps/rejected": -398.506103515625, + "loss": 0.5758, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.0579442977905273, + "rewards/margins": 0.4511980414390564, + "rewards/rejected": -1.5091423988342285, + "step": 1450 + }, + { + "epoch": 0.1, + "learning_rate": 4.774362328319163e-06, + "logits/chosen": -1.8313878774642944, + "logits/rejected": -1.699540376663208, + "logps/chosen": -297.5150146484375, + "logps/rejected": -293.62945556640625, + "loss": 0.6199, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.8172494173049927, + "rewards/margins": 0.26063865423202515, + "rewards/rejected": -1.077888011932373, + "step": 1460 + }, + { + "epoch": 0.1, + "learning_rate": 4.807063440156966e-06, + "logits/chosen": -1.8282153606414795, + "logits/rejected": -1.6260473728179932, + "logps/chosen": -334.1802673339844, + "logps/rejected": -323.95416259765625, + "loss": 0.5395, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.708707869052887, + "rewards/margins": 0.5114428400993347, + "rewards/rejected": -1.2201507091522217, + "step": 1470 + }, + { + "epoch": 0.1, + "learning_rate": 4.839764551994769e-06, + "logits/chosen": -1.8896303176879883, + "logits/rejected": -1.6281471252441406, + "logps/chosen": -348.66851806640625, + "logps/rejected": -361.35980224609375, + "loss": 0.5617, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9996001124382019, + "rewards/margins": 0.5042487382888794, + "rewards/rejected": -1.503848910331726, + "step": 1480 + }, + { + "epoch": 0.1, + "learning_rate": 4.872465663832571e-06, + "logits/chosen": -1.7976363897323608, + "logits/rejected": -1.6468493938446045, + "logps/chosen": -278.28057861328125, + "logps/rejected": -313.6100769042969, + "loss": 0.5226, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.7052208185195923, + "rewards/margins": 0.5578960180282593, + "rewards/rejected": -1.2631168365478516, + "step": 1490 + }, + { + "epoch": 0.1, + "learning_rate": 4.905166775670373e-06, + "logits/chosen": -1.8637592792510986, + "logits/rejected": -1.6277945041656494, + "logps/chosen": -289.73187255859375, + "logps/rejected": -310.935546875, + "loss": 0.5512, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.679719090461731, + "rewards/margins": 0.48597225546836853, + "rewards/rejected": -1.1656912565231323, + "step": 1500 + }, + { + "epoch": 0.1, + "eval_logits/chosen": -1.829469919204712, + "eval_logits/rejected": -1.6728575229644775, + "eval_logps/chosen": -315.8946228027344, + "eval_logps/rejected": -349.49725341796875, + "eval_loss": 0.5626789331436157, + "eval_rewards/accuracies": 0.6370000243186951, + "eval_rewards/chosen": -0.8388968706130981, + "eval_rewards/margins": 0.5399565100669861, + "eval_rewards/rejected": -1.3788533210754395, + "eval_runtime": 736.2443, + "eval_samples_per_second": 2.716, + "eval_steps_per_second": 1.358, + "step": 1500 + }, + { + "epoch": 0.1, + "learning_rate": 4.9378678875081756e-06, + "logits/chosen": -1.9396146535873413, + "logits/rejected": -1.689638376235962, + "logps/chosen": -305.39373779296875, + "logps/rejected": -330.161865234375, + "loss": 0.5266, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.9065009951591492, + "rewards/margins": 0.5577627420425415, + "rewards/rejected": -1.464263677597046, + "step": 1510 + }, + { + "epoch": 0.1, + "learning_rate": 4.9705689993459784e-06, + "logits/chosen": -1.8057773113250732, + "logits/rejected": -1.4475606679916382, + "logps/chosen": -376.16839599609375, + "logps/rejected": -374.25018310546875, + "loss": 0.5563, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7124303579330444, + "rewards/margins": 0.5579492449760437, + "rewards/rejected": -2.2703795433044434, + "step": 1520 + }, + { + "epoch": 0.1, + "learning_rate": 4.999999934793849e-06, + "logits/chosen": -1.771942377090454, + "logits/rejected": -1.6722147464752197, + "logps/chosen": -446.30889892578125, + "logps/rejected": -480.4701232910156, + "loss": 0.5871, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.983555793762207, + "rewards/margins": 0.6357766389846802, + "rewards/rejected": -2.6193320751190186, + "step": 1530 + }, + { + "epoch": 0.1, + "learning_rate": 4.999992110059814e-06, + "logits/chosen": -1.6630691289901733, + "logits/rejected": -1.6542637348175049, + "logps/chosen": -494.4562072753906, + "logps/rejected": -510.5018615722656, + "loss": 0.6301, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.167686939239502, + "rewards/margins": 0.3615902066230774, + "rewards/rejected": -2.5292773246765137, + "step": 1540 + }, + { + "epoch": 0.1, + "learning_rate": 4.999971244142299e-06, + "logits/chosen": -1.7780059576034546, + "logits/rejected": -1.522735357284546, + "logps/chosen": -485.382080078125, + "logps/rejected": -506.91070556640625, + "loss": 0.5534, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1192898750305176, + "rewards/margins": 0.6077495217323303, + "rewards/rejected": -2.727038860321045, + "step": 1550 + }, + { + "epoch": 0.1, + "learning_rate": 4.999937337150149e-06, + "logits/chosen": -1.4890727996826172, + "logits/rejected": -1.411381721496582, + "logps/chosen": -386.9769287109375, + "logps/rejected": -438.4666442871094, + "loss": 0.5325, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.5246206521987915, + "rewards/margins": 0.6687048673629761, + "rewards/rejected": -2.1933255195617676, + "step": 1560 + }, + { + "epoch": 0.1, + "learning_rate": 4.99989038926024e-06, + "logits/chosen": -1.4228270053863525, + "logits/rejected": -1.507315993309021, + "logps/chosen": -337.02008056640625, + "logps/rejected": -390.37542724609375, + "loss": 0.5921, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3859201669692993, + "rewards/margins": 0.5115332007408142, + "rewards/rejected": -1.8974533081054688, + "step": 1570 + }, + { + "epoch": 0.1, + "learning_rate": 4.999830400717476e-06, + "logits/chosen": -1.558600664138794, + "logits/rejected": -1.4721577167510986, + "logps/chosen": -425.5896911621094, + "logps/rejected": -467.05419921875, + "loss": 0.5434, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.3820263147354126, + "rewards/margins": 0.5773999094963074, + "rewards/rejected": -1.9594262838363647, + "step": 1580 + }, + { + "epoch": 0.1, + "learning_rate": 4.999757371834787e-06, + "logits/chosen": -1.4094642400741577, + "logits/rejected": -1.4305822849273682, + "logps/chosen": -366.4520263671875, + "logps/rejected": -418.969482421875, + "loss": 0.5355, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.2663311958312988, + "rewards/margins": 0.6803295016288757, + "rewards/rejected": -1.9466607570648193, + "step": 1590 + }, + { + "epoch": 0.1, + "learning_rate": 4.999671302993125e-06, + "logits/chosen": -1.4092227220535278, + "logits/rejected": -1.3802907466888428, + "logps/chosen": -370.32659912109375, + "logps/rejected": -419.1243591308594, + "loss": 0.6386, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2174619436264038, + "rewards/margins": 0.3340964615345001, + "rewards/rejected": -1.5515583753585815, + "step": 1600 + }, + { + "epoch": 0.1, + "eval_logits/chosen": -1.6499887704849243, + "eval_logits/rejected": -1.501013994216919, + "eval_logps/chosen": -314.13006591796875, + "eval_logps/rejected": -340.3790283203125, + "eval_loss": 0.5758016705513, + "eval_rewards/accuracies": 0.6244999766349792, + "eval_rewards/chosen": -0.8212510943412781, + "eval_rewards/margins": 0.4664204716682434, + "eval_rewards/rejected": -1.2876713275909424, + "eval_runtime": 731.6603, + "eval_samples_per_second": 2.734, + "eval_steps_per_second": 1.367, + "step": 1600 + }, + { + "epoch": 0.11, + "learning_rate": 4.999572194641471e-06, + "logits/chosen": -1.6551891565322876, + "logits/rejected": -1.4926649332046509, + "logps/chosen": -364.64666748046875, + "logps/rejected": -372.8708801269531, + "loss": 0.5305, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.8912836313247681, + "rewards/margins": 0.606941819190979, + "rewards/rejected": -1.4982253313064575, + "step": 1610 + }, + { + "epoch": 0.11, + "learning_rate": 4.999460047296819e-06, + "logits/chosen": -1.593339204788208, + "logits/rejected": -1.4939100742340088, + "logps/chosen": -325.66314697265625, + "logps/rejected": -356.7687683105469, + "loss": 0.5458, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.1005899906158447, + "rewards/margins": 0.5201475620269775, + "rewards/rejected": -1.6207374334335327, + "step": 1620 + }, + { + "epoch": 0.11, + "learning_rate": 4.999334861544186e-06, + "logits/chosen": -1.6162506341934204, + "logits/rejected": -1.380138635635376, + "logps/chosen": -368.8914489746094, + "logps/rejected": -376.7461242675781, + "loss": 0.548, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3943818807601929, + "rewards/margins": 0.6385797262191772, + "rewards/rejected": -2.03296160697937, + "step": 1630 + }, + { + "epoch": 0.11, + "learning_rate": 4.999196638036604e-06, + "logits/chosen": -1.5795015096664429, + "logits/rejected": -1.4061323404312134, + "logps/chosen": -481.0608825683594, + "logps/rejected": -469.82843017578125, + "loss": 0.6616, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.9364579916000366, + "rewards/margins": 0.24003927409648895, + "rewards/rejected": -2.176497459411621, + "step": 1640 + }, + { + "epoch": 0.11, + "learning_rate": 4.999045377495111e-06, + "logits/chosen": -1.1616097688674927, + "logits/rejected": -1.364842176437378, + "logps/chosen": -354.5358581542969, + "logps/rejected": -512.4725341796875, + "loss": 0.5295, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8020122051239014, + "rewards/margins": 0.701541006565094, + "rewards/rejected": -2.5035529136657715, + "step": 1650 + }, + { + "epoch": 0.11, + "learning_rate": 4.998881080708759e-06, + "logits/chosen": -1.1293063163757324, + "logits/rejected": -1.1073631048202515, + "logps/chosen": -438.1863708496094, + "logps/rejected": -431.01885986328125, + "loss": 0.6636, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.9786484241485596, + "rewards/margins": 0.23777297139167786, + "rewards/rejected": -2.216421365737915, + "step": 1660 + }, + { + "epoch": 0.11, + "learning_rate": 4.998703748534599e-06, + "logits/chosen": -0.9836093187332153, + "logits/rejected": -0.8401772379875183, + "logps/chosen": -413.32708740234375, + "logps/rejected": -433.08270263671875, + "loss": 0.5362, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7269728183746338, + "rewards/margins": 0.7044621109962463, + "rewards/rejected": -2.4314348697662354, + "step": 1670 + }, + { + "epoch": 0.11, + "learning_rate": 4.998513381897683e-06, + "logits/chosen": -1.4278907775878906, + "logits/rejected": -1.1960400342941284, + "logps/chosen": -346.00494384765625, + "logps/rejected": -325.62591552734375, + "loss": 0.6458, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.0676120519638062, + "rewards/margins": 0.3661525547504425, + "rewards/rejected": -1.4337646961212158, + "step": 1680 + }, + { + "epoch": 0.11, + "learning_rate": 4.9983099817910565e-06, + "logits/chosen": -1.4905847311019897, + "logits/rejected": -1.3855290412902832, + "logps/chosen": -337.5746154785156, + "logps/rejected": -381.63043212890625, + "loss": 0.5957, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.9231271743774414, + "rewards/margins": 0.4853379726409912, + "rewards/rejected": -1.408465027809143, + "step": 1690 + }, + { + "epoch": 0.11, + "learning_rate": 4.998093549275754e-06, + "logits/chosen": -1.5985859632492065, + "logits/rejected": -1.6034762859344482, + "logps/chosen": -314.97607421875, + "logps/rejected": -382.0792541503906, + "loss": 0.5515, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5464205741882324, + "rewards/margins": 0.4531020224094391, + "rewards/rejected": -0.9995225667953491, + "step": 1700 + }, + { + "epoch": 0.11, + "eval_logits/chosen": -1.6102489233016968, + "eval_logits/rejected": -1.4650778770446777, + "eval_logps/chosen": -293.721435546875, + "eval_logps/rejected": -316.3880615234375, + "eval_loss": 0.5788753628730774, + "eval_rewards/accuracies": 0.6154999732971191, + "eval_rewards/chosen": -0.6171649694442749, + "eval_rewards/margins": 0.43059632182121277, + "eval_rewards/rejected": -1.04776132106781, + "eval_runtime": 732.9705, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.364, + "step": 1700 + }, + { + "epoch": 0.11, + "learning_rate": 4.997864085480794e-06, + "logits/chosen": -1.642925500869751, + "logits/rejected": -1.489255666732788, + "logps/chosen": -360.6557922363281, + "logps/rejected": -397.36541748046875, + "loss": 0.555, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.9430961608886719, + "rewards/margins": 0.5524653196334839, + "rewards/rejected": -1.4955614805221558, + "step": 1710 + }, + { + "epoch": 0.11, + "learning_rate": 4.997621591603171e-06, + "logits/chosen": -1.5001728534698486, + "logits/rejected": -1.3734763860702515, + "logps/chosen": -254.2823486328125, + "logps/rejected": -298.25885009765625, + "loss": 0.6253, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.9624954462051392, + "rewards/margins": 0.44661301374435425, + "rewards/rejected": -1.4091084003448486, + "step": 1720 + }, + { + "epoch": 0.11, + "learning_rate": 4.997366068907853e-06, + "logits/chosen": -1.6290733814239502, + "logits/rejected": -1.5759193897247314, + "logps/chosen": -319.1598205566406, + "logps/rejected": -339.3824768066406, + "loss": 0.5875, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.6075009107589722, + "rewards/margins": 0.4009774327278137, + "rewards/rejected": -1.0084784030914307, + "step": 1730 + }, + { + "epoch": 0.11, + "learning_rate": 4.997097518727771e-06, + "logits/chosen": -1.764134168624878, + "logits/rejected": -1.5227423906326294, + "logps/chosen": -287.41790771484375, + "logps/rejected": -303.2652893066406, + "loss": 0.5696, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.6446150541305542, + "rewards/margins": 0.48969754576683044, + "rewards/rejected": -1.134312391281128, + "step": 1740 + }, + { + "epoch": 0.11, + "learning_rate": 4.9968159424638155e-06, + "logits/chosen": -1.5703229904174805, + "logits/rejected": -1.6936891078948975, + "logps/chosen": -315.3788757324219, + "logps/rejected": -401.99713134765625, + "loss": 0.6687, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.9777250289916992, + "rewards/margins": 0.3477313220500946, + "rewards/rejected": -1.3254563808441162, + "step": 1750 + }, + { + "epoch": 0.12, + "learning_rate": 4.9965213415848235e-06, + "logits/chosen": -1.5106548070907593, + "logits/rejected": -1.2070127725601196, + "logps/chosen": -344.24603271484375, + "logps/rejected": -365.4908447265625, + "loss": 0.5699, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.1584149599075317, + "rewards/margins": 0.5810474753379822, + "rewards/rejected": -1.7394624948501587, + "step": 1760 + }, + { + "epoch": 0.12, + "learning_rate": 4.9962137176275805e-06, + "logits/chosen": -1.5711520910263062, + "logits/rejected": -1.3876466751098633, + "logps/chosen": -352.61053466796875, + "logps/rejected": -397.92626953125, + "loss": 0.5467, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.1831510066986084, + "rewards/margins": 0.458737313747406, + "rewards/rejected": -1.6418883800506592, + "step": 1770 + }, + { + "epoch": 0.12, + "learning_rate": 4.9958930721968015e-06, + "logits/chosen": -1.467863917350769, + "logits/rejected": -1.585448980331421, + "logps/chosen": -384.6903076171875, + "logps/rejected": -428.30322265625, + "loss": 0.6408, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7265514135360718, + "rewards/margins": 0.32111164927482605, + "rewards/rejected": -2.0476629734039307, + "step": 1780 + }, + { + "epoch": 0.12, + "learning_rate": 4.995559406965132e-06, + "logits/chosen": -1.654897928237915, + "logits/rejected": -1.418758749961853, + "logps/chosen": -423.865478515625, + "logps/rejected": -449.65704345703125, + "loss": 0.6173, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9195009469985962, + "rewards/margins": 0.5540726184844971, + "rewards/rejected": -2.473573923110962, + "step": 1790 + }, + { + "epoch": 0.12, + "learning_rate": 4.995212723673131e-06, + "logits/chosen": -1.7198116779327393, + "logits/rejected": -1.5079858303070068, + "logps/chosen": -350.39337158203125, + "logps/rejected": -338.1083068847656, + "loss": 0.5693, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2019023895263672, + "rewards/margins": 0.37697863578796387, + "rewards/rejected": -1.5788812637329102, + "step": 1800 + }, + { + "epoch": 0.12, + "eval_logits/chosen": -1.7295727729797363, + "eval_logits/rejected": -1.5710750818252563, + "eval_logps/chosen": -323.40234375, + "eval_logps/rejected": -346.465087890625, + "eval_loss": 0.5636827349662781, + "eval_rewards/accuracies": 0.6434999704360962, + "eval_rewards/chosen": -0.9139736890792847, + "eval_rewards/margins": 0.43455827236175537, + "eval_rewards/rejected": -1.34853196144104, + "eval_runtime": 732.2426, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.366, + "step": 1800 + }, + { + "epoch": 0.12, + "learning_rate": 4.99485302412927e-06, + "logits/chosen": -1.3826578855514526, + "logits/rejected": -1.382262110710144, + "logps/chosen": -298.8088684082031, + "logps/rejected": -364.51837158203125, + "loss": 0.4857, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9079780578613281, + "rewards/margins": 0.6501679420471191, + "rewards/rejected": -1.5581459999084473, + "step": 1810 + }, + { + "epoch": 0.12, + "learning_rate": 4.994480310209918e-06, + "logits/chosen": -1.5383714437484741, + "logits/rejected": -1.622907280921936, + "logps/chosen": -388.2715759277344, + "logps/rejected": -460.01556396484375, + "loss": 0.5395, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.462885856628418, + "rewards/margins": 0.5798969864845276, + "rewards/rejected": -2.042782783508301, + "step": 1820 + }, + { + "epoch": 0.12, + "learning_rate": 4.994094583859332e-06, + "logits/chosen": -1.383913516998291, + "logits/rejected": -1.2517662048339844, + "logps/chosen": -339.15753173828125, + "logps/rejected": -435.96826171875, + "loss": 0.5427, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.742448091506958, + "rewards/margins": 0.6478117108345032, + "rewards/rejected": -2.3902597427368164, + "step": 1830 + }, + { + "epoch": 0.12, + "learning_rate": 4.9936958470896525e-06, + "logits/chosen": -1.315035104751587, + "logits/rejected": -1.1094564199447632, + "logps/chosen": -418.587158203125, + "logps/rejected": -475.2015686035156, + "loss": 0.42, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.9484198093414307, + "rewards/margins": 0.9951755404472351, + "rewards/rejected": -2.9435954093933105, + "step": 1840 + }, + { + "epoch": 0.12, + "learning_rate": 4.993284101980883e-06, + "logits/chosen": -1.1157817840576172, + "logits/rejected": -1.0088772773742676, + "logps/chosen": -598.3966064453125, + "logps/rejected": -658.9550170898438, + "loss": 0.504, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -3.5590121746063232, + "rewards/margins": 1.0019128322601318, + "rewards/rejected": -4.560925483703613, + "step": 1850 + }, + { + "epoch": 0.12, + "learning_rate": 4.9928593506808885e-06, + "logits/chosen": -1.3301093578338623, + "logits/rejected": -1.1362839937210083, + "logps/chosen": -572.7133178710938, + "logps/rejected": -599.54150390625, + "loss": 0.6122, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -3.158989191055298, + "rewards/margins": 0.6380549669265747, + "rewards/rejected": -3.797044038772583, + "step": 1860 + }, + { + "epoch": 0.12, + "learning_rate": 4.992421595405381e-06, + "logits/chosen": -1.4222590923309326, + "logits/rejected": -1.1619681119918823, + "logps/chosen": -456.4795837402344, + "logps/rejected": -450.6554260253906, + "loss": 0.6322, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.282871723175049, + "rewards/margins": 0.650227427482605, + "rewards/rejected": -2.9330992698669434, + "step": 1870 + }, + { + "epoch": 0.12, + "learning_rate": 4.991970838437905e-06, + "logits/chosen": -1.4716317653656006, + "logits/rejected": -1.406322956085205, + "logps/chosen": -450.2798767089844, + "logps/rejected": -538.4368896484375, + "loss": 0.5038, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.2495033740997314, + "rewards/margins": 0.6005233526229858, + "rewards/rejected": -2.8500266075134277, + "step": 1880 + }, + { + "epoch": 0.12, + "learning_rate": 4.9915070821298294e-06, + "logits/chosen": -1.499276876449585, + "logits/rejected": -1.278843879699707, + "logps/chosen": -382.90435791015625, + "logps/rejected": -406.1724548339844, + "loss": 0.6374, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.1593451499938965, + "rewards/margins": 0.32356566190719604, + "rewards/rejected": -2.4829111099243164, + "step": 1890 + }, + { + "epoch": 0.12, + "learning_rate": 4.991030328900336e-06, + "logits/chosen": -1.4514752626419067, + "logits/rejected": -1.212705373764038, + "logps/chosen": -459.39501953125, + "logps/rejected": -476.75286865234375, + "loss": 0.4312, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7482576370239258, + "rewards/margins": 0.8739405870437622, + "rewards/rejected": -2.6221985816955566, + "step": 1900 + }, + { + "epoch": 0.12, + "eval_logits/chosen": -1.4911623001098633, + "eval_logits/rejected": -1.3446063995361328, + "eval_logps/chosen": -395.8936462402344, + "eval_logps/rejected": -431.74383544921875, + "eval_loss": 0.5713217854499817, + "eval_rewards/accuracies": 0.6299999952316284, + "eval_rewards/chosen": -1.6388871669769287, + "eval_rewards/margins": 0.5624324679374695, + "eval_rewards/rejected": -2.201319694519043, + "eval_runtime": 732.3148, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.366, + "step": 1900 + }, + { + "epoch": 0.12, + "learning_rate": 4.9905405812364014e-06, + "logits/chosen": -1.4818050861358643, + "logits/rejected": -1.4596812725067139, + "logps/chosen": -368.3716125488281, + "logps/rejected": -425.0055236816406, + "loss": 0.551, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.660753607749939, + "rewards/margins": 0.5416861772537231, + "rewards/rejected": -2.202439785003662, + "step": 1910 + }, + { + "epoch": 0.13, + "learning_rate": 4.990037841692791e-06, + "logits/chosen": -1.4347656965255737, + "logits/rejected": -1.2928135395050049, + "logps/chosen": -370.4066467285156, + "logps/rejected": -383.96466064453125, + "loss": 0.5827, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.6436573266983032, + "rewards/margins": 0.5684258341789246, + "rewards/rejected": -2.212083339691162, + "step": 1920 + }, + { + "epoch": 0.13, + "learning_rate": 4.989522112892039e-06, + "logits/chosen": -1.4713729619979858, + "logits/rejected": -1.4406095743179321, + "logps/chosen": -374.61846923828125, + "logps/rejected": -415.1582946777344, + "loss": 0.5761, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.769890546798706, + "rewards/margins": 0.4142988622188568, + "rewards/rejected": -2.184189558029175, + "step": 1930 + }, + { + "epoch": 0.13, + "learning_rate": 4.98899339752444e-06, + "logits/chosen": -1.5092216730117798, + "logits/rejected": -1.2862017154693604, + "logps/chosen": -367.26861572265625, + "logps/rejected": -412.562744140625, + "loss": 0.5197, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3766629695892334, + "rewards/margins": 0.74237459897995, + "rewards/rejected": -2.119037628173828, + "step": 1940 + }, + { + "epoch": 0.13, + "learning_rate": 4.988451698348033e-06, + "logits/chosen": -1.470228672027588, + "logits/rejected": -1.5228326320648193, + "logps/chosen": -328.7142333984375, + "logps/rejected": -400.4761962890625, + "loss": 0.5722, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.5038667917251587, + "rewards/margins": 0.5149255990982056, + "rewards/rejected": -2.0187923908233643, + "step": 1950 + }, + { + "epoch": 0.13, + "learning_rate": 4.987897018188585e-06, + "logits/chosen": -1.424580693244934, + "logits/rejected": -1.2031456232070923, + "logps/chosen": -408.1636047363281, + "logps/rejected": -384.49310302734375, + "loss": 0.7038, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.8554420471191406, + "rewards/margins": 0.33005625009536743, + "rewards/rejected": -2.1854987144470215, + "step": 1960 + }, + { + "epoch": 0.13, + "learning_rate": 4.9873293599395814e-06, + "logits/chosen": -1.628864049911499, + "logits/rejected": -1.5045515298843384, + "logps/chosen": -337.1258850097656, + "logps/rejected": -382.0345458984375, + "loss": 0.5581, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.4369412660598755, + "rewards/margins": 0.48892107605934143, + "rewards/rejected": -1.9258620738983154, + "step": 1970 + }, + { + "epoch": 0.13, + "learning_rate": 4.986748726562203e-06, + "logits/chosen": -1.6803842782974243, + "logits/rejected": -1.5705522298812866, + "logps/chosen": -337.2610778808594, + "logps/rejected": -366.073486328125, + "loss": 0.5317, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2140662670135498, + "rewards/margins": 0.5370473861694336, + "rewards/rejected": -1.7511136531829834, + "step": 1980 + }, + { + "epoch": 0.13, + "learning_rate": 4.98615512108532e-06, + "logits/chosen": -1.6891971826553345, + "logits/rejected": -1.6144077777862549, + "logps/chosen": -342.71551513671875, + "logps/rejected": -398.6066589355469, + "loss": 0.5161, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.2120475769042969, + "rewards/margins": 0.5410778522491455, + "rewards/rejected": -1.753125548362732, + "step": 1990 + }, + { + "epoch": 0.13, + "learning_rate": 4.985548546605469e-06, + "logits/chosen": -1.2585963010787964, + "logits/rejected": -1.388717532157898, + "logps/chosen": -449.9461975097656, + "logps/rejected": -512.4083251953125, + "loss": 0.6104, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.368403911590576, + "rewards/margins": 0.4482155740261078, + "rewards/rejected": -2.816619396209717, + "step": 2000 + }, + { + "epoch": 0.13, + "eval_logits/chosen": -1.3215383291244507, + "eval_logits/rejected": -1.1864279508590698, + "eval_logps/chosen": -490.3330993652344, + "eval_logps/rejected": -524.0951538085938, + "eval_loss": 0.5691915154457092, + "eval_rewards/accuracies": 0.6294999718666077, + "eval_rewards/chosen": -2.5832810401916504, + "eval_rewards/margins": 0.5415511131286621, + "eval_rewards/rejected": -3.1248319149017334, + "eval_runtime": 735.4501, + "eval_samples_per_second": 2.719, + "eval_steps_per_second": 1.36, + "step": 2000 + }, + { + "epoch": 0.13, + "learning_rate": 4.984929006286838e-06, + "logits/chosen": -1.1429121494293213, + "logits/rejected": -1.0560420751571655, + "logps/chosen": -457.6143493652344, + "logps/rejected": -486.137939453125, + "loss": 0.7274, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.483003616333008, + "rewards/margins": 0.250672847032547, + "rewards/rejected": -2.7336764335632324, + "step": 2010 + }, + { + "epoch": 0.13, + "learning_rate": 4.984296503361256e-06, + "logits/chosen": -1.2648851871490479, + "logits/rejected": -1.080210566520691, + "logps/chosen": -455.17901611328125, + "logps/rejected": -455.192626953125, + "loss": 0.5704, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.5325915813446045, + "rewards/margins": 0.4201744496822357, + "rewards/rejected": -2.952765941619873, + "step": 2020 + }, + { + "epoch": 0.13, + "learning_rate": 4.9836510411281645e-06, + "logits/chosen": -1.1588724851608276, + "logits/rejected": -1.0907584428787231, + "logps/chosen": -537.9210815429688, + "logps/rejected": -580.5594482421875, + "loss": 0.4492, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.593892812728882, + "rewards/margins": 0.8204075694084167, + "rewards/rejected": -3.4143004417419434, + "step": 2030 + }, + { + "epoch": 0.13, + "learning_rate": 4.982992622954613e-06, + "logits/chosen": -1.459470510482788, + "logits/rejected": -1.2297568321228027, + "logps/chosen": -494.07342529296875, + "logps/rejected": -436.62872314453125, + "loss": 0.5452, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.1054527759552, + "rewards/margins": 0.5672376155853271, + "rewards/rejected": -2.6726901531219482, + "step": 2040 + }, + { + "epoch": 0.13, + "learning_rate": 4.9823212522752325e-06, + "logits/chosen": -1.6404403448104858, + "logits/rejected": -1.4471931457519531, + "logps/chosen": -450.13055419921875, + "logps/rejected": -485.72357177734375, + "loss": 0.5531, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.723168134689331, + "rewards/margins": 0.696290135383606, + "rewards/rejected": -2.4194586277008057, + "step": 2050 + }, + { + "epoch": 0.13, + "learning_rate": 4.981636932592222e-06, + "logits/chosen": -1.469670057296753, + "logits/rejected": -1.3739349842071533, + "logps/chosen": -332.8818054199219, + "logps/rejected": -400.9057312011719, + "loss": 0.5048, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.1996879577636719, + "rewards/margins": 0.720448911190033, + "rewards/rejected": -1.92013680934906, + "step": 2060 + }, + { + "epoch": 0.14, + "learning_rate": 4.980939667475328e-06, + "logits/chosen": -1.6971523761749268, + "logits/rejected": -1.40958833694458, + "logps/chosen": -414.5674743652344, + "logps/rejected": -404.3069152832031, + "loss": 0.5837, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.403923511505127, + "rewards/margins": 0.5033141374588013, + "rewards/rejected": -1.9072374105453491, + "step": 2070 + }, + { + "epoch": 0.14, + "learning_rate": 4.980229460561826e-06, + "logits/chosen": -1.5918484926223755, + "logits/rejected": -1.5012832880020142, + "logps/chosen": -341.13470458984375, + "logps/rejected": -422.94451904296875, + "loss": 0.4334, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.230533480644226, + "rewards/margins": 0.9711629152297974, + "rewards/rejected": -2.2016961574554443, + "step": 2080 + }, + { + "epoch": 0.14, + "learning_rate": 4.979506315556503e-06, + "logits/chosen": -1.5313241481781006, + "logits/rejected": -1.2765053510665894, + "logps/chosen": -427.59906005859375, + "logps/rejected": -468.064208984375, + "loss": 0.4989, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.4110695123672485, + "rewards/margins": 0.9145315885543823, + "rewards/rejected": -2.325601100921631, + "step": 2090 + }, + { + "epoch": 0.14, + "learning_rate": 4.9787702362316395e-06, + "logits/chosen": -1.6753005981445312, + "logits/rejected": -1.8308302164077759, + "logps/chosen": -316.5, + "logps/rejected": -395.70782470703125, + "loss": 0.589, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2567278146743774, + "rewards/margins": 0.5369380116462708, + "rewards/rejected": -1.7936656475067139, + "step": 2100 + }, + { + "epoch": 0.14, + "eval_logits/chosen": -1.6258045434951782, + "eval_logits/rejected": -1.468246340751648, + "eval_logps/chosen": -352.625732421875, + "eval_logps/rejected": -400.03143310546875, + "eval_loss": 0.554843008518219, + "eval_rewards/accuracies": 0.6355000138282776, + "eval_rewards/chosen": -1.2062077522277832, + "eval_rewards/margins": 0.6779874563217163, + "eval_rewards/rejected": -1.88419508934021, + "eval_runtime": 735.5271, + "eval_samples_per_second": 2.719, + "eval_steps_per_second": 1.36, + "step": 2100 + }, + { + "epoch": 0.14, + "learning_rate": 4.9780212264269835e-06, + "logits/chosen": -1.4886839389801025, + "logits/rejected": -1.3000301122665405, + "logps/chosen": -330.2353210449219, + "logps/rejected": -356.14984130859375, + "loss": 0.6792, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.45416259765625, + "rewards/margins": 0.43050187826156616, + "rewards/rejected": -1.88466477394104, + "step": 2110 + }, + { + "epoch": 0.14, + "learning_rate": 4.977259290049739e-06, + "logits/chosen": -1.6070921421051025, + "logits/rejected": -1.1900372505187988, + "logps/chosen": -418.1002502441406, + "logps/rejected": -486.99951171875, + "loss": 0.3545, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.3115779161453247, + "rewards/margins": 1.3841772079467773, + "rewards/rejected": -2.6957552433013916, + "step": 2120 + }, + { + "epoch": 0.14, + "learning_rate": 4.976484431074538e-06, + "logits/chosen": -1.1471256017684937, + "logits/rejected": -1.1589750051498413, + "logps/chosen": -361.8846130371094, + "logps/rejected": -428.1923828125, + "loss": 0.5364, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.6807912588119507, + "rewards/margins": 0.8391014337539673, + "rewards/rejected": -2.5198922157287598, + "step": 2130 + }, + { + "epoch": 0.14, + "learning_rate": 4.975696653543425e-06, + "logits/chosen": -1.278565764427185, + "logits/rejected": -1.1102664470672607, + "logps/chosen": -451.95819091796875, + "logps/rejected": -545.0523071289062, + "loss": 0.4936, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0230939388275146, + "rewards/margins": 1.0566326379776, + "rewards/rejected": -3.079726457595825, + "step": 2140 + }, + { + "epoch": 0.14, + "learning_rate": 4.974895961565835e-06, + "logits/chosen": -1.0780308246612549, + "logits/rejected": -0.8999558687210083, + "logps/chosen": -351.4781188964844, + "logps/rejected": -439.71697998046875, + "loss": 0.5233, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7622718811035156, + "rewards/margins": 0.8301091194152832, + "rewards/rejected": -2.592381000518799, + "step": 2150 + }, + { + "epoch": 0.14, + "learning_rate": 4.974082359318566e-06, + "logits/chosen": -1.0910061597824097, + "logits/rejected": -1.0057199001312256, + "logps/chosen": -397.6961669921875, + "logps/rejected": -429.198974609375, + "loss": 0.4968, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.4404613971710205, + "rewards/margins": 0.7698196172714233, + "rewards/rejected": -2.2102808952331543, + "step": 2160 + }, + { + "epoch": 0.14, + "learning_rate": 4.973255851045769e-06, + "logits/chosen": -1.1554844379425049, + "logits/rejected": -1.1484893560409546, + "logps/chosen": -364.1314392089844, + "logps/rejected": -411.07427978515625, + "loss": 0.5358, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.4677320718765259, + "rewards/margins": 0.83690345287323, + "rewards/rejected": -2.3046352863311768, + "step": 2170 + }, + { + "epoch": 0.14, + "learning_rate": 4.972416441058915e-06, + "logits/chosen": -0.9132987260818481, + "logits/rejected": -0.8012576103210449, + "logps/chosen": -434.9048767089844, + "logps/rejected": -511.2308044433594, + "loss": 0.4525, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.0514910221099854, + "rewards/margins": 1.0296775102615356, + "rewards/rejected": -3.0811686515808105, + "step": 2180 + }, + { + "epoch": 0.14, + "learning_rate": 4.971564133736777e-06, + "logits/chosen": -0.8563289642333984, + "logits/rejected": -0.6923235654830933, + "logps/chosen": -347.26776123046875, + "logps/rejected": -463.20440673828125, + "loss": 0.4265, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6926708221435547, + "rewards/margins": 1.1095556020736694, + "rewards/rejected": -2.8022265434265137, + "step": 2190 + }, + { + "epoch": 0.14, + "learning_rate": 4.970698933525409e-06, + "logits/chosen": -1.288993000984192, + "logits/rejected": -1.045488953590393, + "logps/chosen": -470.0204162597656, + "logps/rejected": -507.50372314453125, + "loss": 0.632, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9067342281341553, + "rewards/margins": 0.7715495228767395, + "rewards/rejected": -2.67828369140625, + "step": 2200 + }, + { + "epoch": 0.14, + "eval_logits/chosen": -1.086236834526062, + "eval_logits/rejected": -0.9608862400054932, + "eval_logps/chosen": -404.18316650390625, + "eval_logps/rejected": -461.1840515136719, + "eval_loss": 0.5549870729446411, + "eval_rewards/accuracies": 0.6340000033378601, + "eval_rewards/chosen": -1.7217819690704346, + "eval_rewards/margins": 0.7739396095275879, + "eval_rewards/rejected": -2.4957213401794434, + "eval_runtime": 733.4615, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.363, + "step": 2200 + }, + { + "epoch": 0.14, + "learning_rate": 4.969820844938118e-06, + "logits/chosen": -1.1498113870620728, + "logits/rejected": -0.9385069012641907, + "logps/chosen": -386.1590881347656, + "logps/rejected": -409.1542053222656, + "loss": 0.5245, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6355583667755127, + "rewards/margins": 0.7689772844314575, + "rewards/rejected": -2.4045357704162598, + "step": 2210 + }, + { + "epoch": 0.15, + "learning_rate": 4.968929872555444e-06, + "logits/chosen": -0.7596795558929443, + "logits/rejected": -0.6876972913742065, + "logps/chosen": -450.9970703125, + "logps/rejected": -542.7660522460938, + "loss": 0.5959, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.3386266231536865, + "rewards/margins": 0.5610593557357788, + "rewards/rejected": -2.899685859680176, + "step": 2220 + }, + { + "epoch": 0.15, + "learning_rate": 4.968026021025137e-06, + "logits/chosen": -1.260619878768921, + "logits/rejected": -1.0677745342254639, + "logps/chosen": -378.0556335449219, + "logps/rejected": -438.29425048828125, + "loss": 0.4583, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7649071216583252, + "rewards/margins": 1.003040075302124, + "rewards/rejected": -2.76794695854187, + "step": 2230 + }, + { + "epoch": 0.15, + "learning_rate": 4.967109295062128e-06, + "logits/chosen": -1.1158151626586914, + "logits/rejected": -1.0317364931106567, + "logps/chosen": -394.54669189453125, + "logps/rejected": -475.63983154296875, + "loss": 0.5701, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.695452094078064, + "rewards/margins": 0.6017216444015503, + "rewards/rejected": -2.2971739768981934, + "step": 2240 + }, + { + "epoch": 0.15, + "learning_rate": 4.966179699448509e-06, + "logits/chosen": -1.0546188354492188, + "logits/rejected": -0.8983756303787231, + "logps/chosen": -342.80010986328125, + "logps/rejected": -383.49835205078125, + "loss": 0.5724, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.5204514265060425, + "rewards/margins": 0.5852440595626831, + "rewards/rejected": -2.1056954860687256, + "step": 2250 + }, + { + "epoch": 0.15, + "learning_rate": 4.965237239033506e-06, + "logits/chosen": -1.2999796867370605, + "logits/rejected": -1.1046022176742554, + "logps/chosen": -478.2933044433594, + "logps/rejected": -566.3145141601562, + "loss": 0.4039, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.886345624923706, + "rewards/margins": 1.3280307054519653, + "rewards/rejected": -3.214376449584961, + "step": 2260 + }, + { + "epoch": 0.15, + "learning_rate": 4.964281918733453e-06, + "logits/chosen": -1.1871646642684937, + "logits/rejected": -1.0754516124725342, + "logps/chosen": -340.6067810058594, + "logps/rejected": -444.7334899902344, + "loss": 0.4687, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.5749232769012451, + "rewards/margins": 1.0858345031738281, + "rewards/rejected": -2.6607577800750732, + "step": 2270 + }, + { + "epoch": 0.15, + "learning_rate": 4.9633137435317715e-06, + "logits/chosen": -1.2950812578201294, + "logits/rejected": -0.8824887275695801, + "logps/chosen": -359.2813720703125, + "logps/rejected": -409.513916015625, + "loss": 0.4435, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.3638633489608765, + "rewards/margins": 1.1801769733428955, + "rewards/rejected": -2.5440402030944824, + "step": 2280 + }, + { + "epoch": 0.15, + "learning_rate": 4.9623327184789355e-06, + "logits/chosen": -1.4587196111679077, + "logits/rejected": -1.4261622428894043, + "logps/chosen": -321.40521240234375, + "logps/rejected": -359.53515625, + "loss": 0.6161, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.0366475582122803, + "rewards/margins": 0.42956238985061646, + "rewards/rejected": -1.4662100076675415, + "step": 2290 + }, + { + "epoch": 0.15, + "learning_rate": 4.9613388486924525e-06, + "logits/chosen": -1.0783612728118896, + "logits/rejected": -1.239875316619873, + "logps/chosen": -268.2691650390625, + "logps/rejected": -347.37420654296875, + "loss": 0.5211, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.881351113319397, + "rewards/margins": 0.6172493696212769, + "rewards/rejected": -1.498600721359253, + "step": 2300 + }, + { + "epoch": 0.15, + "eval_logits/chosen": -1.415626049041748, + "eval_logits/rejected": -1.2698040008544922, + "eval_logps/chosen": -328.3125915527344, + "eval_logps/rejected": -375.56829833984375, + "eval_loss": 0.5417385697364807, + "eval_rewards/accuracies": 0.637499988079071, + "eval_rewards/chosen": -0.9630764126777649, + "eval_rewards/margins": 0.6764876842498779, + "eval_rewards/rejected": -1.6395639181137085, + "eval_runtime": 731.6375, + "eval_samples_per_second": 2.734, + "eval_steps_per_second": 1.367, + "step": 2300 + }, + { + "epoch": 0.15, + "learning_rate": 4.960332139356834e-06, + "logits/chosen": -1.3438931703567505, + "logits/rejected": -1.1559361219406128, + "logps/chosen": -311.91070556640625, + "logps/rejected": -364.8311767578125, + "loss": 0.482, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.0309386253356934, + "rewards/margins": 0.7936037182807922, + "rewards/rejected": -1.8245422840118408, + "step": 2310 + }, + { + "epoch": 0.15, + "learning_rate": 4.95931259572357e-06, + "logits/chosen": -1.1916639804840088, + "logits/rejected": -1.0088059902191162, + "logps/chosen": -383.302490234375, + "logps/rejected": -496.6766052246094, + "loss": 0.526, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.490625023841858, + "rewards/margins": 0.8770058751106262, + "rewards/rejected": -2.3676304817199707, + "step": 2320 + }, + { + "epoch": 0.15, + "learning_rate": 4.9582802231111e-06, + "logits/chosen": -1.0349695682525635, + "logits/rejected": -1.0374188423156738, + "logps/chosen": -386.32452392578125, + "logps/rejected": -437.10174560546875, + "loss": 0.507, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7445039749145508, + "rewards/margins": 0.7765570282936096, + "rewards/rejected": -2.521061420440674, + "step": 2330 + }, + { + "epoch": 0.15, + "learning_rate": 4.957235026904782e-06, + "logits/chosen": -1.1867823600769043, + "logits/rejected": -0.9589727520942688, + "logps/chosen": -424.76202392578125, + "logps/rejected": -452.0746154785156, + "loss": 0.5258, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.6260957717895508, + "rewards/margins": 0.7842779159545898, + "rewards/rejected": -2.4103739261627197, + "step": 2340 + }, + { + "epoch": 0.15, + "learning_rate": 4.956177012556875e-06, + "logits/chosen": -1.2704133987426758, + "logits/rejected": -1.0597444772720337, + "logps/chosen": -461.6448669433594, + "logps/rejected": -458.13299560546875, + "loss": 0.58, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.17452073097229, + "rewards/margins": 0.6805617213249207, + "rewards/rejected": -2.8550829887390137, + "step": 2350 + }, + { + "epoch": 0.15, + "learning_rate": 4.9551061855864976e-06, + "logits/chosen": -0.8694220781326294, + "logits/rejected": -0.9116750955581665, + "logps/chosen": -376.36865234375, + "logps/rejected": -435.46441650390625, + "loss": 0.562, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.8354053497314453, + "rewards/margins": 0.548851490020752, + "rewards/rejected": -2.384256601333618, + "step": 2360 + }, + { + "epoch": 0.16, + "learning_rate": 4.95402255157961e-06, + "logits/chosen": -1.0590431690216064, + "logits/rejected": -1.0453912019729614, + "logps/chosen": -326.3560485839844, + "logps/rejected": -460.03277587890625, + "loss": 0.5646, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.4129979610443115, + "rewards/margins": 0.6913048624992371, + "rewards/rejected": -2.1043028831481934, + "step": 2370 + }, + { + "epoch": 0.16, + "learning_rate": 4.952926116188977e-06, + "logits/chosen": -1.3469972610473633, + "logits/rejected": -1.3643994331359863, + "logps/chosen": -325.8696594238281, + "logps/rejected": -389.6376037597656, + "loss": 0.6367, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.4249229431152344, + "rewards/margins": 0.24563416838645935, + "rewards/rejected": -1.6705570220947266, + "step": 2380 + }, + { + "epoch": 0.16, + "learning_rate": 4.951816885134143e-06, + "logits/chosen": -1.2549324035644531, + "logits/rejected": -1.3286545276641846, + "logps/chosen": -336.4953308105469, + "logps/rejected": -380.0245361328125, + "loss": 0.5888, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3494236469268799, + "rewards/margins": 0.443899542093277, + "rewards/rejected": -1.7933231592178345, + "step": 2390 + }, + { + "epoch": 0.16, + "learning_rate": 4.950694864201399e-06, + "logits/chosen": -1.2524391412734985, + "logits/rejected": -1.1568235158920288, + "logps/chosen": -386.32135009765625, + "logps/rejected": -451.83154296875, + "loss": 0.4854, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.476393699645996, + "rewards/margins": 0.6338094472885132, + "rewards/rejected": -2.110203266143799, + "step": 2400 + }, + { + "epoch": 0.16, + "eval_logits/chosen": -1.2359693050384521, + "eval_logits/rejected": -1.1046502590179443, + "eval_logps/chosen": -374.9135437011719, + "eval_logps/rejected": -417.5105285644531, + "eval_loss": 0.5439416170120239, + "eval_rewards/accuracies": 0.640500009059906, + "eval_rewards/chosen": -1.429085612297058, + "eval_rewards/margins": 0.6299006342887878, + "eval_rewards/rejected": -2.0589864253997803, + "eval_runtime": 733.4368, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.363, + "step": 2400 + }, + { + "epoch": 0.16, + "learning_rate": 4.9495600592437575e-06, + "logits/chosen": -1.240966558456421, + "logits/rejected": -1.223757028579712, + "logps/chosen": -396.80218505859375, + "logps/rejected": -469.73724365234375, + "loss": 0.4975, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.6990772485733032, + "rewards/margins": 0.6975347399711609, + "rewards/rejected": -2.3966119289398193, + "step": 2410 + }, + { + "epoch": 0.16, + "learning_rate": 4.948412476180917e-06, + "logits/chosen": -0.9485028386116028, + "logits/rejected": -0.7619783878326416, + "logps/chosen": -390.2615661621094, + "logps/rejected": -470.1646423339844, + "loss": 0.4895, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.061856985092163, + "rewards/margins": 1.0263975858688354, + "rewards/rejected": -3.088254451751709, + "step": 2420 + }, + { + "epoch": 0.16, + "learning_rate": 4.947252120999232e-06, + "logits/chosen": -0.9852539896965027, + "logits/rejected": -0.7578643560409546, + "logps/chosen": -502.28619384765625, + "logps/rejected": -482.0196228027344, + "loss": 0.7179, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.351813316345215, + "rewards/margins": 0.3850732445716858, + "rewards/rejected": -2.736886501312256, + "step": 2430 + }, + { + "epoch": 0.16, + "learning_rate": 4.946078999751683e-06, + "logits/chosen": -0.9459400177001953, + "logits/rejected": -0.7881075739860535, + "logps/chosen": -340.50762939453125, + "logps/rejected": -397.1718444824219, + "loss": 0.476, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6088050603866577, + "rewards/margins": 0.7822041511535645, + "rewards/rejected": -2.3910090923309326, + "step": 2440 + }, + { + "epoch": 0.16, + "learning_rate": 4.944893118557847e-06, + "logits/chosen": -0.9686307907104492, + "logits/rejected": -0.880931556224823, + "logps/chosen": -377.506103515625, + "logps/rejected": -386.24383544921875, + "loss": 0.5469, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7102149724960327, + "rewards/margins": 0.6242817640304565, + "rewards/rejected": -2.3344969749450684, + "step": 2450 + }, + { + "epoch": 0.16, + "learning_rate": 4.943694483603861e-06, + "logits/chosen": -1.3755087852478027, + "logits/rejected": -1.023921251296997, + "logps/chosen": -330.78143310546875, + "logps/rejected": -368.04034423828125, + "loss": 0.4777, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.0477737188339233, + "rewards/margins": 0.8823789358139038, + "rewards/rejected": -1.9301526546478271, + "step": 2460 + }, + { + "epoch": 0.16, + "learning_rate": 4.9424831011423914e-06, + "logits/chosen": -1.3603510856628418, + "logits/rejected": -1.3010962009429932, + "logps/chosen": -402.3771057128906, + "logps/rejected": -410.086181640625, + "loss": 0.6041, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.1395056247711182, + "rewards/margins": 0.44723695516586304, + "rewards/rejected": -1.5867425203323364, + "step": 2470 + }, + { + "epoch": 0.16, + "learning_rate": 4.9412589774926015e-06, + "logits/chosen": -1.409360647201538, + "logits/rejected": -1.1556938886642456, + "logps/chosen": -383.52313232421875, + "logps/rejected": -408.6988525390625, + "loss": 0.532, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.0933339595794678, + "rewards/margins": 0.757992148399353, + "rewards/rejected": -1.8513259887695312, + "step": 2480 + }, + { + "epoch": 0.16, + "learning_rate": 4.940022119040121e-06, + "logits/chosen": -1.3208924531936646, + "logits/rejected": -1.1365283727645874, + "logps/chosen": -410.063720703125, + "logps/rejected": -431.35003662109375, + "loss": 0.593, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.181477665901184, + "rewards/margins": 0.49104517698287964, + "rewards/rejected": -1.6725225448608398, + "step": 2490 + }, + { + "epoch": 0.16, + "learning_rate": 4.93877253223701e-06, + "logits/chosen": -1.073589563369751, + "logits/rejected": -1.0274441242218018, + "logps/chosen": -459.6949768066406, + "logps/rejected": -509.46380615234375, + "loss": 0.4768, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6742274761199951, + "rewards/margins": 0.8134153485298157, + "rewards/rejected": -2.487642765045166, + "step": 2500 + }, + { + "epoch": 0.16, + "eval_logits/chosen": -0.9927495121955872, + "eval_logits/rejected": -0.8693163990974426, + "eval_logps/chosen": -433.1884460449219, + "eval_logps/rejected": -486.5682373046875, + "eval_loss": 0.5401906967163086, + "eval_rewards/accuracies": 0.6359999775886536, + "eval_rewards/chosen": -2.0118348598480225, + "eval_rewards/margins": 0.7377288937568665, + "eval_rewards/rejected": -2.749563455581665, + "eval_runtime": 732.9007, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.364, + "step": 2500 + }, + { + "epoch": 0.16, + "learning_rate": 4.937510223601725e-06, + "logits/chosen": -1.2273633480072021, + "logits/rejected": -1.1791640520095825, + "logps/chosen": -452.0333557128906, + "logps/rejected": -472.3573303222656, + "loss": 0.5202, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.895073652267456, + "rewards/margins": 0.6253536939620972, + "rewards/rejected": -2.5204272270202637, + "step": 2510 + }, + { + "epoch": 0.16, + "learning_rate": 4.936235199719085e-06, + "logits/chosen": -0.900462806224823, + "logits/rejected": -0.8040259480476379, + "logps/chosen": -408.2585144042969, + "logps/rejected": -480.28875732421875, + "loss": 0.4679, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.359565258026123, + "rewards/margins": 1.022094488143921, + "rewards/rejected": -3.381659746170044, + "step": 2520 + }, + { + "epoch": 0.17, + "learning_rate": 4.93494746724024e-06, + "logits/chosen": -0.9940454363822937, + "logits/rejected": -0.8842470049858093, + "logps/chosen": -448.1106872558594, + "logps/rejected": -553.7257080078125, + "loss": 0.462, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.2430620193481445, + "rewards/margins": 0.8815500140190125, + "rewards/rejected": -3.1246120929718018, + "step": 2530 + }, + { + "epoch": 0.17, + "learning_rate": 4.933647032882635e-06, + "logits/chosen": -1.041330099105835, + "logits/rejected": -0.8723998069763184, + "logps/chosen": -496.6675720214844, + "logps/rejected": -534.0340576171875, + "loss": 0.5552, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.518056869506836, + "rewards/margins": 0.8178606033325195, + "rewards/rejected": -3.3359177112579346, + "step": 2540 + }, + { + "epoch": 0.17, + "learning_rate": 4.932333903429969e-06, + "logits/chosen": -0.698738694190979, + "logits/rejected": -0.5703662633895874, + "logps/chosen": -446.97027587890625, + "logps/rejected": -452.32757568359375, + "loss": 0.7093, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -2.5388777256011963, + "rewards/margins": 0.3116335868835449, + "rewards/rejected": -2.850511312484741, + "step": 2550 + }, + { + "epoch": 0.17, + "learning_rate": 4.931008085732172e-06, + "logits/chosen": -0.9040409326553345, + "logits/rejected": -0.5525091886520386, + "logps/chosen": -459.99798583984375, + "logps/rejected": -480.68487548828125, + "loss": 0.4977, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.5717508792877197, + "rewards/margins": 0.7068825364112854, + "rewards/rejected": -3.2786335945129395, + "step": 2560 + }, + { + "epoch": 0.17, + "learning_rate": 4.9296695867053565e-06, + "logits/chosen": -1.0348608493804932, + "logits/rejected": -0.7598494291305542, + "logps/chosen": -526.0467529296875, + "logps/rejected": -582.1028442382812, + "loss": 0.4165, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.2544517517089844, + "rewards/margins": 1.220160722732544, + "rewards/rejected": -3.4746127128601074, + "step": 2570 + }, + { + "epoch": 0.17, + "learning_rate": 4.928318413331791e-06, + "logits/chosen": -1.0867879390716553, + "logits/rejected": -1.023646593093872, + "logps/chosen": -433.85919189453125, + "logps/rejected": -448.56402587890625, + "loss": 0.6846, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.226320743560791, + "rewards/margins": 0.30690130591392517, + "rewards/rejected": -2.533221960067749, + "step": 2580 + }, + { + "epoch": 0.17, + "learning_rate": 4.926954572659855e-06, + "logits/chosen": -0.9278467297554016, + "logits/rejected": -0.8537686467170715, + "logps/chosen": -448.48760986328125, + "logps/rejected": -546.9074096679688, + "loss": 0.4983, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0629193782806396, + "rewards/margins": 0.8123389482498169, + "rewards/rejected": -2.875257968902588, + "step": 2590 + }, + { + "epoch": 0.17, + "learning_rate": 4.925578071804013e-06, + "logits/chosen": -0.8431898355484009, + "logits/rejected": -0.8421382904052734, + "logps/chosen": -424.267578125, + "logps/rejected": -553.5916748046875, + "loss": 0.562, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.94561767578125, + "rewards/margins": 0.7191228866577148, + "rewards/rejected": -2.6647403240203857, + "step": 2600 + }, + { + "epoch": 0.17, + "eval_logits/chosen": -1.0128614902496338, + "eval_logits/rejected": -0.891107439994812, + "eval_logps/chosen": -433.5695495605469, + "eval_logps/rejected": -486.4391174316406, + "eval_loss": 0.527816891670227, + "eval_rewards/accuracies": 0.6604999899864197, + "eval_rewards/chosen": -2.01564621925354, + "eval_rewards/margins": 0.7326260209083557, + "eval_rewards/rejected": -2.748272180557251, + "eval_runtime": 732.9757, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.364, + "step": 2600 + }, + { + "epoch": 0.17, + "learning_rate": 4.924188917944763e-06, + "logits/chosen": -1.1159459352493286, + "logits/rejected": -0.8869976997375488, + "logps/chosen": -421.93389892578125, + "logps/rejected": -481.2115783691406, + "loss": 0.491, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0192878246307373, + "rewards/margins": 0.9279811978340149, + "rewards/rejected": -2.9472689628601074, + "step": 2610 + }, + { + "epoch": 0.17, + "learning_rate": 4.922787118328617e-06, + "logits/chosen": -1.141663908958435, + "logits/rejected": -0.7913933396339417, + "logps/chosen": -453.99383544921875, + "logps/rejected": -407.4873046875, + "loss": 0.6754, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -2.2397615909576416, + "rewards/margins": 0.2980630099773407, + "rewards/rejected": -2.537824869155884, + "step": 2620 + }, + { + "epoch": 0.17, + "learning_rate": 4.921372680268045e-06, + "logits/chosen": -0.980755627155304, + "logits/rejected": -0.9280126690864563, + "logps/chosen": -459.7035217285156, + "logps/rejected": -448.31048583984375, + "loss": 0.7411, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.296988010406494, + "rewards/margins": 0.18975761532783508, + "rewards/rejected": -2.486745834350586, + "step": 2630 + }, + { + "epoch": 0.17, + "learning_rate": 4.919945611141451e-06, + "logits/chosen": -1.341280460357666, + "logits/rejected": -1.0558829307556152, + "logps/chosen": -359.87152099609375, + "logps/rejected": -346.2890930175781, + "loss": 0.5557, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3463690280914307, + "rewards/margins": 0.44936901330947876, + "rewards/rejected": -1.795737862586975, + "step": 2640 + }, + { + "epoch": 0.17, + "learning_rate": 4.918505918393125e-06, + "logits/chosen": -1.2368559837341309, + "logits/rejected": -1.1329363584518433, + "logps/chosen": -267.66839599609375, + "logps/rejected": -348.6911315917969, + "loss": 0.5407, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.9863770604133606, + "rewards/margins": 0.5764933824539185, + "rewards/rejected": -1.5628705024719238, + "step": 2650 + }, + { + "epoch": 0.17, + "learning_rate": 4.91705360953321e-06, + "logits/chosen": -1.203028917312622, + "logits/rejected": -1.117807388305664, + "logps/chosen": -373.701416015625, + "logps/rejected": -405.60211181640625, + "loss": 0.4926, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.2821382284164429, + "rewards/margins": 0.6455599665641785, + "rewards/rejected": -1.9276981353759766, + "step": 2660 + }, + { + "epoch": 0.17, + "learning_rate": 4.9155886921376615e-06, + "logits/chosen": -1.1834170818328857, + "logits/rejected": -1.1657710075378418, + "logps/chosen": -341.26007080078125, + "logps/rejected": -403.8941955566406, + "loss": 0.5907, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3529164791107178, + "rewards/margins": 0.4096578061580658, + "rewards/rejected": -1.7625744342803955, + "step": 2670 + }, + { + "epoch": 0.18, + "learning_rate": 4.914111173848205e-06, + "logits/chosen": -1.3189375400543213, + "logits/rejected": -1.2658817768096924, + "logps/chosen": -385.70587158203125, + "logps/rejected": -400.50225830078125, + "loss": 0.6556, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.526742935180664, + "rewards/margins": 0.302143394947052, + "rewards/rejected": -1.8288863897323608, + "step": 2680 + }, + { + "epoch": 0.18, + "learning_rate": 4.9126210623723e-06, + "logits/chosen": -1.1100072860717773, + "logits/rejected": -1.185240626335144, + "logps/chosen": -326.9442138671875, + "logps/rejected": -431.2759704589844, + "loss": 0.5286, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.2657082080841064, + "rewards/margins": 0.7055976390838623, + "rewards/rejected": -1.9713056087493896, + "step": 2690 + }, + { + "epoch": 0.18, + "learning_rate": 4.911118365483098e-06, + "logits/chosen": -1.2706750631332397, + "logits/rejected": -1.2281222343444824, + "logps/chosen": -346.92291259765625, + "logps/rejected": -448.357177734375, + "loss": 0.4748, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.451686978340149, + "rewards/margins": 0.9200803637504578, + "rewards/rejected": -2.371767520904541, + "step": 2700 + }, + { + "epoch": 0.18, + "eval_logits/chosen": -1.2759194374084473, + "eval_logits/rejected": -1.1406419277191162, + "eval_logps/chosen": -376.82635498046875, + "eval_logps/rejected": -422.0545349121094, + "eval_loss": 0.5314846038818359, + "eval_rewards/accuracies": 0.6514999866485596, + "eval_rewards/chosen": -1.4482139348983765, + "eval_rewards/margins": 0.6562120914459229, + "eval_rewards/rejected": -2.1044256687164307, + "eval_runtime": 733.3385, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.364, + "step": 2700 + }, + { + "epoch": 0.18, + "learning_rate": 4.909603091019403e-06, + "logits/chosen": -1.401475191116333, + "logits/rejected": -1.179467797279358, + "logps/chosen": -364.406494140625, + "logps/rejected": -411.4903259277344, + "loss": 0.482, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2802919149398804, + "rewards/margins": 0.8370407223701477, + "rewards/rejected": -2.1173324584960938, + "step": 2710 + }, + { + "epoch": 0.18, + "learning_rate": 4.908075246885626e-06, + "logits/chosen": -1.1405667066574097, + "logits/rejected": -1.1133387088775635, + "logps/chosen": -269.01739501953125, + "logps/rejected": -277.61370849609375, + "loss": 0.6395, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.1828992366790771, + "rewards/margins": 0.35987579822540283, + "rewards/rejected": -1.5427749156951904, + "step": 2720 + }, + { + "epoch": 0.18, + "learning_rate": 4.906534841051755e-06, + "logits/chosen": -1.0903264284133911, + "logits/rejected": -1.0795084238052368, + "logps/chosen": -362.6637268066406, + "logps/rejected": -419.4922790527344, + "loss": 0.6301, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.1926027536392212, + "rewards/margins": 0.45792943239212036, + "rewards/rejected": -1.6505321264266968, + "step": 2730 + }, + { + "epoch": 0.18, + "learning_rate": 4.904981881553297e-06, + "logits/chosen": -1.340993881225586, + "logits/rejected": -1.1244676113128662, + "logps/chosen": -337.9717102050781, + "logps/rejected": -317.94488525390625, + "loss": 0.5715, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.137753963470459, + "rewards/margins": 0.4409133791923523, + "rewards/rejected": -1.578667402267456, + "step": 2740 + }, + { + "epoch": 0.18, + "learning_rate": 4.903416376491252e-06, + "logits/chosen": -1.4110819101333618, + "logits/rejected": -1.1954363584518433, + "logps/chosen": -390.9253845214844, + "logps/rejected": -444.5638122558594, + "loss": 0.4639, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.1263313293457031, + "rewards/margins": 0.9423055648803711, + "rewards/rejected": -2.068636894226074, + "step": 2750 + }, + { + "epoch": 0.18, + "learning_rate": 4.90183833403206e-06, + "logits/chosen": -1.5229685306549072, + "logits/rejected": -1.4395592212677002, + "logps/chosen": -385.73980712890625, + "logps/rejected": -420.03094482421875, + "loss": 0.4857, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.2152669429779053, + "rewards/margins": 0.7054417729377747, + "rewards/rejected": -1.9207088947296143, + "step": 2760 + }, + { + "epoch": 0.18, + "learning_rate": 4.900247762407564e-06, + "logits/chosen": -1.0933618545532227, + "logits/rejected": -1.0539687871932983, + "logps/chosen": -344.75738525390625, + "logps/rejected": -438.8707580566406, + "loss": 0.5357, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.724654197692871, + "rewards/margins": 0.7207070589065552, + "rewards/rejected": -2.4453611373901367, + "step": 2770 + }, + { + "epoch": 0.18, + "learning_rate": 4.898644669914965e-06, + "logits/chosen": -1.0270262956619263, + "logits/rejected": -0.965491771697998, + "logps/chosen": -471.7977600097656, + "logps/rejected": -546.3108520507812, + "loss": 0.5477, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.4772531986236572, + "rewards/margins": 0.8833915591239929, + "rewards/rejected": -3.360644578933716, + "step": 2780 + }, + { + "epoch": 0.18, + "learning_rate": 4.897029064916778e-06, + "logits/chosen": -0.7744370698928833, + "logits/rejected": -0.7252274751663208, + "logps/chosen": -438.18341064453125, + "logps/rejected": -465.3837890625, + "loss": 0.7191, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -2.3003344535827637, + "rewards/margins": 0.3958927094936371, + "rewards/rejected": -2.6962273120880127, + "step": 2790 + }, + { + "epoch": 0.18, + "learning_rate": 4.895400955840791e-06, + "logits/chosen": -1.1814029216766357, + "logits/rejected": -0.6611972451210022, + "logps/chosen": -378.6611328125, + "logps/rejected": -414.54052734375, + "loss": 0.5099, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.58309805393219, + "rewards/margins": 0.7757716178894043, + "rewards/rejected": -2.358869791030884, + "step": 2800 + }, + { + "epoch": 0.18, + "eval_logits/chosen": -1.0748944282531738, + "eval_logits/rejected": -0.9484049081802368, + "eval_logps/chosen": -392.2981872558594, + "eval_logps/rejected": -440.3302917480469, + "eval_loss": 0.5305582284927368, + "eval_rewards/accuracies": 0.6549999713897705, + "eval_rewards/chosen": -1.6029322147369385, + "eval_rewards/margins": 0.684251606464386, + "eval_rewards/rejected": -2.2871837615966797, + "eval_runtime": 734.2092, + "eval_samples_per_second": 2.724, + "eval_steps_per_second": 1.362, + "step": 2800 + }, + { + "epoch": 0.18, + "learning_rate": 4.893760351180018e-06, + "logits/chosen": -1.0402562618255615, + "logits/rejected": -1.0653431415557861, + "logps/chosen": -370.8222351074219, + "logps/rejected": -438.2054138183594, + "loss": 0.6082, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.7228095531463623, + "rewards/margins": 0.6228219270706177, + "rewards/rejected": -2.3456313610076904, + "step": 2810 + }, + { + "epoch": 0.18, + "learning_rate": 4.892107259492657e-06, + "logits/chosen": -1.0466301441192627, + "logits/rejected": -0.919894814491272, + "logps/chosen": -394.4199523925781, + "logps/rejected": -442.1200256347656, + "loss": 0.6315, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.5456066131591797, + "rewards/margins": 0.41681140661239624, + "rewards/rejected": -1.9624179601669312, + "step": 2820 + }, + { + "epoch": 0.19, + "learning_rate": 4.890441689402042e-06, + "logits/chosen": -1.3622853755950928, + "logits/rejected": -1.162502408027649, + "logps/chosen": -481.157958984375, + "logps/rejected": -511.90240478515625, + "loss": 0.4618, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.4565303325653076, + "rewards/margins": 0.7874441146850586, + "rewards/rejected": -2.243974208831787, + "step": 2830 + }, + { + "epoch": 0.19, + "learning_rate": 4.888763649596606e-06, + "logits/chosen": -1.2705270051956177, + "logits/rejected": -1.1603574752807617, + "logps/chosen": -331.831298828125, + "logps/rejected": -385.3037109375, + "loss": 0.5833, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2166961431503296, + "rewards/margins": 0.6227377653121948, + "rewards/rejected": -1.839434027671814, + "step": 2840 + }, + { + "epoch": 0.19, + "learning_rate": 4.887073148829824e-06, + "logits/chosen": -1.1340843439102173, + "logits/rejected": -1.0734115839004517, + "logps/chosen": -406.3775939941406, + "logps/rejected": -445.03936767578125, + "loss": 0.5258, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.394740104675293, + "rewards/margins": 0.6159732937812805, + "rewards/rejected": -2.0107131004333496, + "step": 2850 + }, + { + "epoch": 0.19, + "learning_rate": 4.885370195920177e-06, + "logits/chosen": -0.9719009399414062, + "logits/rejected": -0.9220370054244995, + "logps/chosen": -333.6672058105469, + "logps/rejected": -379.9608459472656, + "loss": 0.5954, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.483434796333313, + "rewards/margins": 0.5486684441566467, + "rewards/rejected": -2.0321033000946045, + "step": 2860 + }, + { + "epoch": 0.19, + "learning_rate": 4.883654799751101e-06, + "logits/chosen": -1.0081456899642944, + "logits/rejected": -0.9878345727920532, + "logps/chosen": -335.7508544921875, + "logps/rejected": -447.65374755859375, + "loss": 0.5001, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.200417160987854, + "rewards/margins": 0.7716276049613953, + "rewards/rejected": -1.972044587135315, + "step": 2870 + }, + { + "epoch": 0.19, + "learning_rate": 4.8819269692709435e-06, + "logits/chosen": -1.1583213806152344, + "logits/rejected": -0.9544271230697632, + "logps/chosen": -384.2664489746094, + "logps/rejected": -385.5480651855469, + "loss": 0.5024, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.1722023487091064, + "rewards/margins": 0.736437201499939, + "rewards/rejected": -1.9086395502090454, + "step": 2880 + }, + { + "epoch": 0.19, + "learning_rate": 4.880186713492915e-06, + "logits/chosen": -1.0247581005096436, + "logits/rejected": -0.8900755047798157, + "logps/chosen": -375.8417663574219, + "logps/rejected": -384.86859130859375, + "loss": 0.535, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.4812443256378174, + "rewards/margins": 0.626487672328949, + "rewards/rejected": -2.1077322959899902, + "step": 2890 + }, + { + "epoch": 0.19, + "learning_rate": 4.878434041495041e-06, + "logits/chosen": -1.0221517086029053, + "logits/rejected": -1.1312007904052734, + "logps/chosen": -399.26617431640625, + "logps/rejected": -516.553955078125, + "loss": 0.4184, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.6210047006607056, + "rewards/margins": 1.154254674911499, + "rewards/rejected": -2.775259494781494, + "step": 2900 + }, + { + "epoch": 0.19, + "eval_logits/chosen": -0.907734751701355, + "eval_logits/rejected": -0.7929914593696594, + "eval_logps/chosen": -393.54962158203125, + "eval_logps/rejected": -452.6503601074219, + "eval_loss": 0.5266603827476501, + "eval_rewards/accuracies": 0.6514999866485596, + "eval_rewards/chosen": -1.6154468059539795, + "eval_rewards/margins": 0.7949380874633789, + "eval_rewards/rejected": -2.4103844165802, + "eval_runtime": 730.902, + "eval_samples_per_second": 2.736, + "eval_steps_per_second": 1.368, + "step": 2900 + }, + { + "epoch": 0.19, + "learning_rate": 4.876668962420117e-06, + "logits/chosen": -0.8577669858932495, + "logits/rejected": -0.7146766781806946, + "logps/chosen": -455.3329162597656, + "logps/rejected": -499.1671447753906, + "loss": 0.4407, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.656818151473999, + "rewards/margins": 1.034503698348999, + "rewards/rejected": -2.691321849822998, + "step": 2910 + }, + { + "epoch": 0.19, + "learning_rate": 4.87489148547566e-06, + "logits/chosen": -0.8334639668464661, + "logits/rejected": -0.7265030145645142, + "logps/chosen": -500.32470703125, + "logps/rejected": -542.7216796875, + "loss": 0.6218, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.4046618938446045, + "rewards/margins": 0.7556990385055542, + "rewards/rejected": -3.1603610515594482, + "step": 2920 + }, + { + "epoch": 0.19, + "learning_rate": 4.873101619933862e-06, + "logits/chosen": -0.9287866353988647, + "logits/rejected": -0.6335155367851257, + "logps/chosen": -481.3199768066406, + "logps/rejected": -514.2613525390625, + "loss": 0.5545, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.1679275035858154, + "rewards/margins": 0.8225703239440918, + "rewards/rejected": -2.9904978275299072, + "step": 2930 + }, + { + "epoch": 0.19, + "learning_rate": 4.8712993751315385e-06, + "logits/chosen": -0.9662561416625977, + "logits/rejected": -0.8811322450637817, + "logps/chosen": -273.9098205566406, + "logps/rejected": -336.27252197265625, + "loss": 0.5461, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.5325188636779785, + "rewards/margins": 0.5846083164215088, + "rewards/rejected": -2.1171271800994873, + "step": 2940 + }, + { + "epoch": 0.19, + "learning_rate": 4.869484760470079e-06, + "logits/chosen": -0.9696873426437378, + "logits/rejected": -0.8237133026123047, + "logps/chosen": -332.93359375, + "logps/rejected": -366.3995361328125, + "loss": 0.5814, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.4427276849746704, + "rewards/margins": 0.6976507902145386, + "rewards/rejected": -2.140378475189209, + "step": 2950 + }, + { + "epoch": 0.19, + "learning_rate": 4.867657785415404e-06, + "logits/chosen": -0.9058094024658203, + "logits/rejected": -0.7002671360969543, + "logps/chosen": -431.8790588378906, + "logps/rejected": -483.63006591796875, + "loss": 0.4947, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.8278001546859741, + "rewards/margins": 0.934956431388855, + "rewards/rejected": -2.762756824493408, + "step": 2960 + }, + { + "epoch": 0.19, + "learning_rate": 4.865818459497911e-06, + "logits/chosen": -1.0235707759857178, + "logits/rejected": -0.9009502530097961, + "logps/chosen": -490.59881591796875, + "logps/rejected": -472.571044921875, + "loss": 0.5176, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0642013549804688, + "rewards/margins": 0.6676945686340332, + "rewards/rejected": -2.731895923614502, + "step": 2970 + }, + { + "epoch": 0.19, + "learning_rate": 4.863966792312423e-06, + "logits/chosen": -1.034481167793274, + "logits/rejected": -0.6752454042434692, + "logps/chosen": -434.35595703125, + "logps/rejected": -499.046875, + "loss": 0.4784, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9289706945419312, + "rewards/margins": 1.0574229955673218, + "rewards/rejected": -2.986393690109253, + "step": 2980 + }, + { + "epoch": 0.2, + "learning_rate": 4.862102793518145e-06, + "logits/chosen": -0.956391453742981, + "logits/rejected": -1.0064009428024292, + "logps/chosen": -385.90771484375, + "logps/rejected": -467.2142028808594, + "loss": 0.5787, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9162585735321045, + "rewards/margins": 0.7438151240348816, + "rewards/rejected": -2.660073757171631, + "step": 2990 + }, + { + "epoch": 0.2, + "learning_rate": 4.8602264728386075e-06, + "logits/chosen": -1.1530312299728394, + "logits/rejected": -1.024060845375061, + "logps/chosen": -411.135009765625, + "logps/rejected": -497.8170471191406, + "loss": 0.468, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.586355209350586, + "rewards/margins": 0.9010065197944641, + "rewards/rejected": -2.487361431121826, + "step": 3000 + }, + { + "epoch": 0.2, + "eval_logits/chosen": -1.0168744325637817, + "eval_logits/rejected": -0.8916241526603699, + "eval_logps/chosen": -405.43792724609375, + "eval_logps/rejected": -467.9596252441406, + "eval_loss": 0.5222868919372559, + "eval_rewards/accuracies": 0.6554999947547913, + "eval_rewards/chosen": -1.7343300580978394, + "eval_rewards/margins": 0.8291473388671875, + "eval_rewards/rejected": -2.563477039337158, + "eval_runtime": 733.6976, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 3000 + }, + { + "epoch": 0.2, + "learning_rate": 4.858337840061616e-06, + "logits/chosen": -0.8814601898193359, + "logits/rejected": -0.9170808792114258, + "logps/chosen": -346.5814514160156, + "logps/rejected": -471.98895263671875, + "loss": 0.5828, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.6350104808807373, + "rewards/margins": 0.717583954334259, + "rewards/rejected": -2.3525943756103516, + "step": 3010 + }, + { + "epoch": 0.2, + "learning_rate": 4.856436905039208e-06, + "logits/chosen": -0.8554957509040833, + "logits/rejected": -0.6725913286209106, + "logps/chosen": -409.9076232910156, + "logps/rejected": -458.8953552246094, + "loss": 0.4747, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.987026572227478, + "rewards/margins": 0.8337499499320984, + "rewards/rejected": -2.8207764625549316, + "step": 3020 + }, + { + "epoch": 0.2, + "learning_rate": 4.854523677687588e-06, + "logits/chosen": -0.6897114515304565, + "logits/rejected": -0.8571540713310242, + "logps/chosen": -395.29534912109375, + "logps/rejected": -481.5233459472656, + "loss": 0.5643, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.1404976844787598, + "rewards/margins": 0.6783149242401123, + "rewards/rejected": -2.818812608718872, + "step": 3030 + }, + { + "epoch": 0.2, + "learning_rate": 4.85259816798709e-06, + "logits/chosen": -1.00284743309021, + "logits/rejected": -0.6904066801071167, + "logps/chosen": -503.5082092285156, + "logps/rejected": -529.4007568359375, + "loss": 0.3823, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.157461404800415, + "rewards/margins": 1.0644841194152832, + "rewards/rejected": -3.221945285797119, + "step": 3040 + }, + { + "epoch": 0.2, + "learning_rate": 4.850660385982114e-06, + "logits/chosen": -0.9757447242736816, + "logits/rejected": -0.8276710510253906, + "logps/chosen": -455.947509765625, + "logps/rejected": -474.081298828125, + "loss": 0.5308, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.0822834968566895, + "rewards/margins": 0.7341514825820923, + "rewards/rejected": -2.816434621810913, + "step": 3050 + }, + { + "epoch": 0.2, + "learning_rate": 4.848710341781081e-06, + "logits/chosen": -0.640845000743866, + "logits/rejected": -0.7251681089401245, + "logps/chosen": -449.40155029296875, + "logps/rejected": -478.784912109375, + "loss": 0.6729, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.754999876022339, + "rewards/margins": 0.35110121965408325, + "rewards/rejected": -3.1061012744903564, + "step": 3060 + }, + { + "epoch": 0.2, + "learning_rate": 4.846748045556377e-06, + "logits/chosen": -0.6704039573669434, + "logits/rejected": -0.5354386568069458, + "logps/chosen": -501.2828063964844, + "logps/rejected": -522.3720703125, + "loss": 0.5013, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.6179566383361816, + "rewards/margins": 0.8160024881362915, + "rewards/rejected": -3.4339592456817627, + "step": 3070 + }, + { + "epoch": 0.2, + "learning_rate": 4.8447735075442995e-06, + "logits/chosen": -0.637302577495575, + "logits/rejected": -0.49296456575393677, + "logps/chosen": -482.5350646972656, + "logps/rejected": -590.9215698242188, + "loss": 0.4817, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.8497281074523926, + "rewards/margins": 1.117870569229126, + "rewards/rejected": -3.9675984382629395, + "step": 3080 + }, + { + "epoch": 0.2, + "learning_rate": 4.8427867380450075e-06, + "logits/chosen": -0.9691807627677917, + "logits/rejected": -0.6462706923484802, + "logps/chosen": -459.28887939453125, + "logps/rejected": -493.9776306152344, + "loss": 0.5673, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.2894959449768066, + "rewards/margins": 0.8093854784965515, + "rewards/rejected": -3.098881483078003, + "step": 3090 + }, + { + "epoch": 0.2, + "learning_rate": 4.840787747422462e-06, + "logits/chosen": -1.2492235898971558, + "logits/rejected": -1.0929436683654785, + "logps/chosen": -324.46063232421875, + "logps/rejected": -351.09698486328125, + "loss": 0.5857, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.2880451679229736, + "rewards/margins": 0.5689392685890198, + "rewards/rejected": -1.8569844961166382, + "step": 3100 + }, + { + "epoch": 0.2, + "eval_logits/chosen": -1.2385766506195068, + "eval_logits/rejected": -1.1036633253097534, + "eval_logps/chosen": -358.3794860839844, + "eval_logps/rejected": -410.8307800292969, + "eval_loss": 0.5289512872695923, + "eval_rewards/accuracies": 0.6520000100135803, + "eval_rewards/chosen": -1.2637453079223633, + "eval_rewards/margins": 0.7284430861473083, + "eval_rewards/rejected": -1.9921886920928955, + "eval_runtime": 732.5166, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 3100 + }, + { + "epoch": 0.2, + "learning_rate": 4.838776546104378e-06, + "logits/chosen": -1.2828580141067505, + "logits/rejected": -1.148850679397583, + "logps/chosen": -423.83135986328125, + "logps/rejected": -468.1695861816406, + "loss": 0.4887, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.4287811517715454, + "rewards/margins": 0.8474019169807434, + "rewards/rejected": -2.2761833667755127, + "step": 3110 + }, + { + "epoch": 0.2, + "learning_rate": 4.836753144582168e-06, + "logits/chosen": -1.1283457279205322, + "logits/rejected": -0.7700001001358032, + "logps/chosen": -421.4664611816406, + "logps/rejected": -505.2184143066406, + "loss": 0.443, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7923486232757568, + "rewards/margins": 1.0786457061767578, + "rewards/rejected": -2.8709943294525146, + "step": 3120 + }, + { + "epoch": 0.2, + "learning_rate": 4.834717553410884e-06, + "logits/chosen": -1.1379692554473877, + "logits/rejected": -1.06531822681427, + "logps/chosen": -363.9742126464844, + "logps/rejected": -480.22222900390625, + "loss": 0.5057, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7469819784164429, + "rewards/margins": 1.025788426399231, + "rewards/rejected": -2.7727701663970947, + "step": 3130 + }, + { + "epoch": 0.21, + "learning_rate": 4.832669783209167e-06, + "logits/chosen": -0.9420145750045776, + "logits/rejected": -1.055405616760254, + "logps/chosen": -443.606201171875, + "logps/rejected": -490.388671875, + "loss": 0.6093, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.9918018579483032, + "rewards/margins": 0.46233540773391724, + "rewards/rejected": -2.454137086868286, + "step": 3140 + }, + { + "epoch": 0.21, + "learning_rate": 4.8306098446591895e-06, + "logits/chosen": -0.7147270441055298, + "logits/rejected": -0.7811274528503418, + "logps/chosen": -347.41033935546875, + "logps/rejected": -424.43267822265625, + "loss": 0.5833, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.711414098739624, + "rewards/margins": 0.5209105610847473, + "rewards/rejected": -2.2323248386383057, + "step": 3150 + }, + { + "epoch": 0.21, + "learning_rate": 4.828537748506601e-06, + "logits/chosen": -1.4567492008209229, + "logits/rejected": -1.2312958240509033, + "logps/chosen": -414.09161376953125, + "logps/rejected": -411.62432861328125, + "loss": 0.5788, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.4069335460662842, + "rewards/margins": 0.5377890467643738, + "rewards/rejected": -1.9447225332260132, + "step": 3160 + }, + { + "epoch": 0.21, + "learning_rate": 4.826453505560469e-06, + "logits/chosen": -1.0341746807098389, + "logits/rejected": -1.0024795532226562, + "logps/chosen": -364.7231140136719, + "logps/rejected": -411.63037109375, + "loss": 0.5837, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.7374398708343506, + "rewards/margins": 0.6060377359390259, + "rewards/rejected": -2.343477725982666, + "step": 3170 + }, + { + "epoch": 0.21, + "learning_rate": 4.824357126693226e-06, + "logits/chosen": -1.0167440176010132, + "logits/rejected": -0.9091693162918091, + "logps/chosen": -429.69970703125, + "logps/rejected": -449.07855224609375, + "loss": 0.5235, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7095658779144287, + "rewards/margins": 0.6810619831085205, + "rewards/rejected": -2.39062762260437, + "step": 3180 + }, + { + "epoch": 0.21, + "learning_rate": 4.8222486228406105e-06, + "logits/chosen": -1.2612932920455933, + "logits/rejected": -1.0270341634750366, + "logps/chosen": -406.03485107421875, + "logps/rejected": -476.10479736328125, + "loss": 0.4328, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9479446411132812, + "rewards/margins": 1.0475102663040161, + "rewards/rejected": -2.995454788208008, + "step": 3190 + }, + { + "epoch": 0.21, + "learning_rate": 4.820128005001612e-06, + "logits/chosen": -0.9638387560844421, + "logits/rejected": -0.8402373194694519, + "logps/chosen": -445.51678466796875, + "logps/rejected": -538.071044921875, + "loss": 0.4504, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2652316093444824, + "rewards/margins": 1.1409714221954346, + "rewards/rejected": -3.406203031539917, + "step": 3200 + }, + { + "epoch": 0.21, + "eval_logits/chosen": -1.1166863441467285, + "eval_logits/rejected": -0.9832271337509155, + "eval_logps/chosen": -494.8057861328125, + "eval_logps/rejected": -568.17138671875, + "eval_loss": 0.5196487307548523, + "eval_rewards/accuracies": 0.6514999866485596, + "eval_rewards/chosen": -2.6280083656311035, + "eval_rewards/margins": 0.9375866055488586, + "eval_rewards/rejected": -3.5655946731567383, + "eval_runtime": 732.8446, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.365, + "step": 3200 + }, + { + "epoch": 0.21, + "learning_rate": 4.817995284238412e-06, + "logits/chosen": -0.996303915977478, + "logits/rejected": -1.036203145980835, + "logps/chosen": -439.01483154296875, + "logps/rejected": -553.1556396484375, + "loss": 0.5445, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.4249050617218018, + "rewards/margins": 0.866560161113739, + "rewards/rejected": -3.2914650440216064, + "step": 3210 + }, + { + "epoch": 0.21, + "learning_rate": 4.815850471676327e-06, + "logits/chosen": -1.1847374439239502, + "logits/rejected": -0.9033486247062683, + "logps/chosen": -514.692626953125, + "logps/rejected": -615.4830322265625, + "loss": 0.4885, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.7627861499786377, + "rewards/margins": 1.050376534461975, + "rewards/rejected": -3.8131625652313232, + "step": 3220 + }, + { + "epoch": 0.21, + "learning_rate": 4.813693578503751e-06, + "logits/chosen": -1.0441510677337646, + "logits/rejected": -0.860817551612854, + "logps/chosen": -532.6041870117188, + "logps/rejected": -567.3224487304688, + "loss": 0.4816, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.360973834991455, + "rewards/margins": 0.9017590284347534, + "rewards/rejected": -3.262732982635498, + "step": 3230 + }, + { + "epoch": 0.21, + "learning_rate": 4.811524615972093e-06, + "logits/chosen": -0.7930616140365601, + "logits/rejected": -0.8713744282722473, + "logps/chosen": -501.078125, + "logps/rejected": -601.9039916992188, + "loss": 0.5722, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.7114250659942627, + "rewards/margins": 0.9262357950210571, + "rewards/rejected": -3.6376609802246094, + "step": 3240 + }, + { + "epoch": 0.21, + "learning_rate": 4.809343595395724e-06, + "logits/chosen": -1.4246814250946045, + "logits/rejected": -1.1265549659729004, + "logps/chosen": -415.13128662109375, + "logps/rejected": -465.318359375, + "loss": 0.5769, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.260756254196167, + "rewards/margins": 0.8077578544616699, + "rewards/rejected": -3.068514108657837, + "step": 3250 + }, + { + "epoch": 0.21, + "learning_rate": 4.807150528151918e-06, + "logits/chosen": -1.0579570531845093, + "logits/rejected": -1.0017118453979492, + "logps/chosen": -337.6005554199219, + "logps/rejected": -462.49261474609375, + "loss": 0.4633, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7090799808502197, + "rewards/margins": 1.0702515840530396, + "rewards/rejected": -2.779331684112549, + "step": 3260 + }, + { + "epoch": 0.21, + "learning_rate": 4.804945425680787e-06, + "logits/chosen": -1.1110305786132812, + "logits/rejected": -1.067275881767273, + "logps/chosen": -423.29437255859375, + "logps/rejected": -457.4410095214844, + "loss": 0.5568, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3485116958618164, + "rewards/margins": 0.5422489047050476, + "rewards/rejected": -2.8907601833343506, + "step": 3270 + }, + { + "epoch": 0.21, + "learning_rate": 4.802728299485225e-06, + "logits/chosen": -0.9096890687942505, + "logits/rejected": -0.8615697026252747, + "logps/chosen": -394.5549011230469, + "logps/rejected": -478.35870361328125, + "loss": 0.6697, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.439504384994507, + "rewards/margins": 0.6227213144302368, + "rewards/rejected": -3.0622260570526123, + "step": 3280 + }, + { + "epoch": 0.22, + "learning_rate": 4.8004991611308495e-06, + "logits/chosen": -1.2388406991958618, + "logits/rejected": -1.0739996433258057, + "logps/chosen": -403.4649658203125, + "logps/rejected": -469.5654296875, + "loss": 0.498, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.6191120147705078, + "rewards/margins": 0.8215069770812988, + "rewards/rejected": -2.4406189918518066, + "step": 3290 + }, + { + "epoch": 0.22, + "learning_rate": 4.798258022245937e-06, + "logits/chosen": -1.2814171314239502, + "logits/rejected": -1.0564707517623901, + "logps/chosen": -368.33807373046875, + "logps/rejected": -409.11407470703125, + "loss": 0.5336, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.5073158740997314, + "rewards/margins": 0.7924111485481262, + "rewards/rejected": -2.299726963043213, + "step": 3300 + }, + { + "epoch": 0.22, + "eval_logits/chosen": -1.2265251874923706, + "eval_logits/rejected": -1.0916633605957031, + "eval_logps/chosen": -364.0115051269531, + "eval_logps/rejected": -422.55963134765625, + "eval_loss": 0.521187961101532, + "eval_rewards/accuracies": 0.6514999866485596, + "eval_rewards/chosen": -1.3200657367706299, + "eval_rewards/margins": 0.7894111275672913, + "eval_rewards/rejected": -2.1094770431518555, + "eval_runtime": 730.6664, + "eval_samples_per_second": 2.737, + "eval_steps_per_second": 1.369, + "step": 3300 + }, + { + "epoch": 0.22, + "learning_rate": 4.796004894521365e-06, + "logits/chosen": -1.304469347000122, + "logits/rejected": -1.0183453559875488, + "logps/chosen": -370.98699951171875, + "logps/rejected": -483.6045837402344, + "loss": 0.5777, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.424172282218933, + "rewards/margins": 0.9136108160018921, + "rewards/rejected": -2.337782859802246, + "step": 3310 + }, + { + "epoch": 0.22, + "learning_rate": 4.7937397897105545e-06, + "logits/chosen": -1.218902587890625, + "logits/rejected": -1.1205322742462158, + "logps/chosen": -343.5941467285156, + "logps/rejected": -372.71905517578125, + "loss": 0.5801, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.383612871170044, + "rewards/margins": 0.5433837175369263, + "rewards/rejected": -1.9269965887069702, + "step": 3320 + }, + { + "epoch": 0.22, + "learning_rate": 4.791462719629399e-06, + "logits/chosen": -1.0824165344238281, + "logits/rejected": -1.012694239616394, + "logps/chosen": -321.79803466796875, + "logps/rejected": -382.03997802734375, + "loss": 0.5033, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3607285022735596, + "rewards/margins": 0.8265340924263, + "rewards/rejected": -2.187262773513794, + "step": 3330 + }, + { + "epoch": 0.22, + "learning_rate": 4.789173696156212e-06, + "logits/chosen": -1.1417255401611328, + "logits/rejected": -0.8498822450637817, + "logps/chosen": -445.33349609375, + "logps/rejected": -552.0372314453125, + "loss": 0.3828, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.6889288425445557, + "rewards/margins": 1.2535320520401, + "rewards/rejected": -2.942460536956787, + "step": 3340 + }, + { + "epoch": 0.22, + "learning_rate": 4.786872731231662e-06, + "logits/chosen": -1.1248489618301392, + "logits/rejected": -1.0287617444992065, + "logps/chosen": -451.65765380859375, + "logps/rejected": -502.49786376953125, + "loss": 0.6218, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.3476672172546387, + "rewards/margins": 0.6220924854278564, + "rewards/rejected": -2.969759464263916, + "step": 3350 + }, + { + "epoch": 0.22, + "learning_rate": 4.784559836858709e-06, + "logits/chosen": -0.9597482681274414, + "logits/rejected": -0.7844263911247253, + "logps/chosen": -422.6121520996094, + "logps/rejected": -501.7942810058594, + "loss": 0.4192, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.8869502544403076, + "rewards/margins": 1.0960309505462646, + "rewards/rejected": -2.9829812049865723, + "step": 3360 + }, + { + "epoch": 0.22, + "learning_rate": 4.782235025102542e-06, + "logits/chosen": -0.9880424737930298, + "logits/rejected": -0.9343817830085754, + "logps/chosen": -464.04107666015625, + "logps/rejected": -534.37109375, + "loss": 0.5225, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.3772170543670654, + "rewards/margins": 0.8781053423881531, + "rewards/rejected": -3.2553226947784424, + "step": 3370 + }, + { + "epoch": 0.22, + "learning_rate": 4.779898308090519e-06, + "logits/chosen": -1.0745431184768677, + "logits/rejected": -0.8649323582649231, + "logps/chosen": -502.0604553222656, + "logps/rejected": -583.8115234375, + "loss": 0.4702, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.273831844329834, + "rewards/margins": 1.1858172416687012, + "rewards/rejected": -3.4596495628356934, + "step": 3380 + }, + { + "epoch": 0.22, + "learning_rate": 4.777549698012101e-06, + "logits/chosen": -0.9469674229621887, + "logits/rejected": -0.8883152008056641, + "logps/chosen": -492.7947692871094, + "logps/rejected": -608.96044921875, + "loss": 0.4002, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.495781421661377, + "rewards/margins": 1.3532531261444092, + "rewards/rejected": -3.849034547805786, + "step": 3390 + }, + { + "epoch": 0.22, + "learning_rate": 4.775189207118787e-06, + "logits/chosen": -0.8769720792770386, + "logits/rejected": -0.8017680048942566, + "logps/chosen": -508.2986755371094, + "logps/rejected": -565.0167236328125, + "loss": 0.5781, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.36480975151062, + "rewards/margins": 0.7905575633049011, + "rewards/rejected": -3.155367374420166, + "step": 3400 + }, + { + "epoch": 0.22, + "eval_logits/chosen": -1.0673307180404663, + "eval_logits/rejected": -0.9397266507148743, + "eval_logps/chosen": -407.0195617675781, + "eval_logps/rejected": -473.85296630859375, + "eval_loss": 0.517623782157898, + "eval_rewards/accuracies": 0.6575000286102295, + "eval_rewards/chosen": -1.7501461505889893, + "eval_rewards/margins": 0.8722642064094543, + "eval_rewards/rejected": -2.6224100589752197, + "eval_runtime": 731.9001, + "eval_samples_per_second": 2.733, + "eval_steps_per_second": 1.366, + "step": 3400 + }, + { + "epoch": 0.22, + "learning_rate": 4.772816847724054e-06, + "logits/chosen": -1.276422381401062, + "logits/rejected": -1.2553038597106934, + "logps/chosen": -358.34124755859375, + "logps/rejected": -426.73919677734375, + "loss": 0.6499, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3578667640686035, + "rewards/margins": 0.6890023946762085, + "rewards/rejected": -2.0468690395355225, + "step": 3410 + }, + { + "epoch": 0.22, + "learning_rate": 4.770432632203294e-06, + "logits/chosen": -1.2462594509124756, + "logits/rejected": -1.1783959865570068, + "logps/chosen": -340.1909484863281, + "logps/rejected": -344.7999572753906, + "loss": 0.5608, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.9654380679130554, + "rewards/margins": 0.5492728352546692, + "rewards/rejected": -1.5147110223770142, + "step": 3420 + }, + { + "epoch": 0.22, + "learning_rate": 4.768036572993738e-06, + "logits/chosen": -1.4957689046859741, + "logits/rejected": -1.4557548761367798, + "logps/chosen": -371.65008544921875, + "logps/rejected": -415.08367919921875, + "loss": 0.5477, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9163110852241516, + "rewards/margins": 0.5903352499008179, + "rewards/rejected": -1.5066463947296143, + "step": 3430 + }, + { + "epoch": 0.23, + "learning_rate": 4.765628682594409e-06, + "logits/chosen": -1.5271755456924438, + "logits/rejected": -1.4071924686431885, + "logps/chosen": -328.58270263671875, + "logps/rejected": -370.56390380859375, + "loss": 0.5162, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.8482564091682434, + "rewards/margins": 0.6588364839553833, + "rewards/rejected": -1.5070927143096924, + "step": 3440 + }, + { + "epoch": 0.23, + "learning_rate": 4.763208973566041e-06, + "logits/chosen": -1.331571340560913, + "logits/rejected": -1.2520661354064941, + "logps/chosen": -322.298828125, + "logps/rejected": -401.99053955078125, + "loss": 0.5404, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3816914558410645, + "rewards/margins": 0.6097648739814758, + "rewards/rejected": -1.9914562702178955, + "step": 3450 + }, + { + "epoch": 0.23, + "learning_rate": 4.76077745853102e-06, + "logits/chosen": -1.48663330078125, + "logits/rejected": -1.4448163509368896, + "logps/chosen": -415.95184326171875, + "logps/rejected": -490.90899658203125, + "loss": 0.5291, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6589769124984741, + "rewards/margins": 0.7383204698562622, + "rewards/rejected": -2.3972973823547363, + "step": 3460 + }, + { + "epoch": 0.23, + "learning_rate": 4.758334150173322e-06, + "logits/chosen": -1.3083218336105347, + "logits/rejected": -1.2009087800979614, + "logps/chosen": -372.3233337402344, + "logps/rejected": -440.8482360839844, + "loss": 0.4207, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.0923935174942017, + "rewards/margins": 0.9357854723930359, + "rewards/rejected": -2.0281789302825928, + "step": 3470 + }, + { + "epoch": 0.23, + "learning_rate": 4.755879061238439e-06, + "logits/chosen": -1.3399627208709717, + "logits/rejected": -1.1547447443008423, + "logps/chosen": -425.87567138671875, + "logps/rejected": -493.44903564453125, + "loss": 0.5857, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6970230340957642, + "rewards/margins": 0.8021049499511719, + "rewards/rejected": -2.4991281032562256, + "step": 3480 + }, + { + "epoch": 0.23, + "learning_rate": 4.753412204533317e-06, + "logits/chosen": -1.462085485458374, + "logits/rejected": -0.9828954935073853, + "logps/chosen": -426.71044921875, + "logps/rejected": -479.6546936035156, + "loss": 0.514, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6385246515274048, + "rewards/margins": 0.9833550453186035, + "rewards/rejected": -2.6218795776367188, + "step": 3490 + }, + { + "epoch": 0.23, + "learning_rate": 4.750933592926292e-06, + "logits/chosen": -1.3087482452392578, + "logits/rejected": -0.9995695352554321, + "logps/chosen": -407.4541320800781, + "logps/rejected": -479.6214294433594, + "loss": 0.4228, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.887264609336853, + "rewards/margins": 0.9938368797302246, + "rewards/rejected": -2.881101131439209, + "step": 3500 + }, + { + "epoch": 0.23, + "eval_logits/chosen": -1.1501306295394897, + "eval_logits/rejected": -1.0210661888122559, + "eval_logps/chosen": -404.4117736816406, + "eval_logps/rejected": -466.7912902832031, + "eval_loss": 0.5153305530548096, + "eval_rewards/accuracies": 0.6589999794960022, + "eval_rewards/chosen": -1.7240681648254395, + "eval_rewards/margins": 0.8277252912521362, + "eval_rewards/rejected": -2.5517935752868652, + "eval_runtime": 734.8426, + "eval_samples_per_second": 2.722, + "eval_steps_per_second": 1.361, + "step": 3500 + }, + { + "epoch": 0.23, + "learning_rate": 4.7484432393470124e-06, + "logits/chosen": -1.3669573068618774, + "logits/rejected": -0.9049302339553833, + "logps/chosen": -343.1621398925781, + "logps/rejected": -392.91400146484375, + "loss": 0.4902, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.410801649093628, + "rewards/margins": 1.0514872074127197, + "rewards/rejected": -2.4622886180877686, + "step": 3510 + }, + { + "epoch": 0.23, + "learning_rate": 4.745941156786385e-06, + "logits/chosen": -0.8993209004402161, + "logits/rejected": -0.9887765049934387, + "logps/chosen": -308.3384094238281, + "logps/rejected": -441.26788330078125, + "loss": 0.466, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.552992820739746, + "rewards/margins": 1.0208826065063477, + "rewards/rejected": -2.573875665664673, + "step": 3520 + }, + { + "epoch": 0.23, + "learning_rate": 4.743427358296497e-06, + "logits/chosen": -1.115281343460083, + "logits/rejected": -0.955095112323761, + "logps/chosen": -328.03436279296875, + "logps/rejected": -452.24737548828125, + "loss": 0.4754, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.38326096534729, + "rewards/margins": 1.1290562152862549, + "rewards/rejected": -2.512317419052124, + "step": 3530 + }, + { + "epoch": 0.23, + "learning_rate": 4.740901856990553e-06, + "logits/chosen": -0.9862316846847534, + "logits/rejected": -0.836779773235321, + "logps/chosen": -399.65155029296875, + "logps/rejected": -411.66839599609375, + "loss": 0.6233, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.429596185684204, + "rewards/margins": 0.5608941316604614, + "rewards/rejected": -1.990490198135376, + "step": 3540 + }, + { + "epoch": 0.23, + "learning_rate": 4.738364666042804e-06, + "logits/chosen": -1.303762435913086, + "logits/rejected": -0.9813443422317505, + "logps/chosen": -376.2354736328125, + "logps/rejected": -400.13238525390625, + "loss": 0.4688, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.844904899597168, + "rewards/margins": 0.875367283821106, + "rewards/rejected": -1.7202720642089844, + "step": 3550 + }, + { + "epoch": 0.23, + "learning_rate": 4.735815798688483e-06, + "logits/chosen": -1.18997061252594, + "logits/rejected": -0.980979323387146, + "logps/chosen": -313.696533203125, + "logps/rejected": -416.5016174316406, + "loss": 0.5036, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.153645396232605, + "rewards/margins": 0.8266595005989075, + "rewards/rejected": -1.9803049564361572, + "step": 3560 + }, + { + "epoch": 0.23, + "learning_rate": 4.7332552682237285e-06, + "logits/chosen": -1.060894250869751, + "logits/rejected": -0.7684231996536255, + "logps/chosen": -291.71331787109375, + "logps/rejected": -369.025390625, + "loss": 0.4658, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.1795337200164795, + "rewards/margins": 0.9308455586433411, + "rewards/rejected": -2.1103789806365967, + "step": 3570 + }, + { + "epoch": 0.23, + "learning_rate": 4.7306830880055234e-06, + "logits/chosen": -1.1081796884536743, + "logits/rejected": -1.0646052360534668, + "logps/chosen": -383.7330627441406, + "logps/rejected": -464.07659912109375, + "loss": 0.5619, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9299920797348022, + "rewards/margins": 0.7200648784637451, + "rewards/rejected": -2.650057077407837, + "step": 3580 + }, + { + "epoch": 0.23, + "learning_rate": 4.728099271451619e-06, + "logits/chosen": -1.0864530801773071, + "logits/rejected": -1.1386919021606445, + "logps/chosen": -353.6946716308594, + "logps/rejected": -416.392822265625, + "loss": 0.5043, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.626866102218628, + "rewards/margins": 0.6965071558952332, + "rewards/rejected": -2.323373317718506, + "step": 3590 + }, + { + "epoch": 0.24, + "learning_rate": 4.725503832040466e-06, + "logits/chosen": -0.8184248208999634, + "logits/rejected": -0.6163265705108643, + "logps/chosen": -312.90264892578125, + "logps/rejected": -412.98785400390625, + "loss": 0.5345, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.6374473571777344, + "rewards/margins": 0.7399784326553345, + "rewards/rejected": -2.3774256706237793, + "step": 3600 + }, + { + "epoch": 0.24, + "eval_logits/chosen": -0.8562494516372681, + "eval_logits/rejected": -0.743854820728302, + "eval_logps/chosen": -430.83062744140625, + "eval_logps/rejected": -490.9767150878906, + "eval_loss": 0.5146420001983643, + "eval_rewards/accuracies": 0.6579999923706055, + "eval_rewards/chosen": -1.9882564544677734, + "eval_rewards/margins": 0.8053914904594421, + "eval_rewards/rejected": -2.7936480045318604, + "eval_runtime": 734.536, + "eval_samples_per_second": 2.723, + "eval_steps_per_second": 1.361, + "step": 3600 + }, + { + "epoch": 0.24, + "learning_rate": 4.722896783311152e-06, + "logits/chosen": -0.9259394407272339, + "logits/rejected": -0.7786766290664673, + "logps/chosen": -487.031982421875, + "logps/rejected": -614.58251953125, + "loss": 0.6077, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.279829740524292, + "rewards/margins": 0.764165997505188, + "rewards/rejected": -3.0439953804016113, + "step": 3610 + }, + { + "epoch": 0.24, + "learning_rate": 4.720278138863318e-06, + "logits/chosen": -0.9295158386230469, + "logits/rejected": -0.9707782864570618, + "logps/chosen": -387.5835266113281, + "logps/rejected": -414.12506103515625, + "loss": 0.6133, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9746183156967163, + "rewards/margins": 0.5834863781929016, + "rewards/rejected": -2.5581045150756836, + "step": 3620 + }, + { + "epoch": 0.24, + "learning_rate": 4.717647912357095e-06, + "logits/chosen": -1.2270703315734863, + "logits/rejected": -1.2645610570907593, + "logps/chosen": -470.46527099609375, + "logps/rejected": -520.5504760742188, + "loss": 0.6555, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.979880928993225, + "rewards/margins": 0.3715277314186096, + "rewards/rejected": -2.3514084815979004, + "step": 3630 + }, + { + "epoch": 0.24, + "learning_rate": 4.715006117513035e-06, + "logits/chosen": -1.2985150814056396, + "logits/rejected": -1.1814923286437988, + "logps/chosen": -514.2528076171875, + "logps/rejected": -529.6771240234375, + "loss": 0.4864, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9012712240219116, + "rewards/margins": 0.7060447931289673, + "rewards/rejected": -2.6073157787323, + "step": 3640 + }, + { + "epoch": 0.24, + "learning_rate": 4.7123527681120326e-06, + "logits/chosen": -1.1452562808990479, + "logits/rejected": -0.959757924079895, + "logps/chosen": -443.2506408691406, + "logps/rejected": -499.8810119628906, + "loss": 0.4578, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9607951641082764, + "rewards/margins": 0.8478196859359741, + "rewards/rejected": -2.80861496925354, + "step": 3650 + }, + { + "epoch": 0.24, + "learning_rate": 4.7096878779952594e-06, + "logits/chosen": -1.3756132125854492, + "logits/rejected": -1.319220781326294, + "logps/chosen": -489.84930419921875, + "logps/rejected": -562.1311645507812, + "loss": 0.4599, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.144359588623047, + "rewards/margins": 0.7486457824707031, + "rewards/rejected": -2.89300537109375, + "step": 3660 + }, + { + "epoch": 0.24, + "learning_rate": 4.707011461064086e-06, + "logits/chosen": -1.172327995300293, + "logits/rejected": -0.9126516580581665, + "logps/chosen": -487.898193359375, + "logps/rejected": -538.8970947265625, + "loss": 0.4521, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7903915643692017, + "rewards/margins": 0.951895534992218, + "rewards/rejected": -2.7422873973846436, + "step": 3670 + }, + { + "epoch": 0.24, + "learning_rate": 4.704323531280016e-06, + "logits/chosen": -1.0632213354110718, + "logits/rejected": -0.9930667877197266, + "logps/chosen": -504.03582763671875, + "logps/rejected": -516.4459228515625, + "loss": 0.484, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7682430744171143, + "rewards/margins": 0.9560562372207642, + "rewards/rejected": -2.724299192428589, + "step": 3680 + }, + { + "epoch": 0.24, + "learning_rate": 4.701624102664606e-06, + "logits/chosen": -1.2993719577789307, + "logits/rejected": -1.1282496452331543, + "logps/chosen": -481.0072326660156, + "logps/rejected": -521.73876953125, + "loss": 0.5127, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.198913097381592, + "rewards/margins": 0.9483180046081543, + "rewards/rejected": -3.147231101989746, + "step": 3690 + }, + { + "epoch": 0.24, + "learning_rate": 4.698913189299399e-06, + "logits/chosen": -1.1892201900482178, + "logits/rejected": -1.1501682996749878, + "logps/chosen": -415.744873046875, + "logps/rejected": -520.8284301757812, + "loss": 0.6089, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.2959303855895996, + "rewards/margins": 0.7257683277130127, + "rewards/rejected": -3.0216987133026123, + "step": 3700 + }, + { + "epoch": 0.24, + "eval_logits/chosen": -1.1420601606369019, + "eval_logits/rejected": -1.0099586248397827, + "eval_logps/chosen": -474.0902099609375, + "eval_logps/rejected": -541.633056640625, + "eval_loss": 0.5181785821914673, + "eval_rewards/accuracies": 0.6504999995231628, + "eval_rewards/chosen": -2.4208528995513916, + "eval_rewards/margins": 0.8793590664863586, + "eval_rewards/rejected": -3.3002119064331055, + "eval_runtime": 729.6855, + "eval_samples_per_second": 2.741, + "eval_steps_per_second": 1.37, + "step": 3700 + }, + { + "epoch": 0.24, + "learning_rate": 4.696190805325847e-06, + "logits/chosen": -1.2062838077545166, + "logits/rejected": -1.0717065334320068, + "logps/chosen": -442.6526794433594, + "logps/rejected": -527.166015625, + "loss": 0.4301, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.362738847732544, + "rewards/margins": 1.1163194179534912, + "rewards/rejected": -3.479058027267456, + "step": 3710 + }, + { + "epoch": 0.24, + "learning_rate": 4.693456964945239e-06, + "logits/chosen": -1.227285385131836, + "logits/rejected": -0.876573920249939, + "logps/chosen": -574.7136840820312, + "logps/rejected": -603.5750122070312, + "loss": 0.3663, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.7660810947418213, + "rewards/margins": 1.2995359897613525, + "rewards/rejected": -4.065616607666016, + "step": 3720 + }, + { + "epoch": 0.24, + "learning_rate": 4.6907116824186245e-06, + "logits/chosen": -1.271610975265503, + "logits/rejected": -1.2354601621627808, + "logps/chosen": -478.7208557128906, + "logps/rejected": -537.8218994140625, + "loss": 0.6851, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -2.513932704925537, + "rewards/margins": 0.5929728746414185, + "rewards/rejected": -3.106905937194824, + "step": 3730 + }, + { + "epoch": 0.24, + "learning_rate": 4.687954972066742e-06, + "logits/chosen": -1.354651689529419, + "logits/rejected": -1.2123476266860962, + "logps/chosen": -435.0054626464844, + "logps/rejected": -538.2466430664062, + "loss": 0.4836, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.051891803741455, + "rewards/margins": 1.241121768951416, + "rewards/rejected": -3.293013334274292, + "step": 3740 + }, + { + "epoch": 0.25, + "learning_rate": 4.685186848269944e-06, + "logits/chosen": -1.3648970127105713, + "logits/rejected": -1.206362009048462, + "logps/chosen": -378.1536560058594, + "logps/rejected": -416.2786560058594, + "loss": 0.588, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.6628129482269287, + "rewards/margins": 0.7550392150878906, + "rewards/rejected": -2.4178521633148193, + "step": 3750 + }, + { + "epoch": 0.25, + "learning_rate": 4.682407325468119e-06, + "logits/chosen": -1.3208792209625244, + "logits/rejected": -1.028571367263794, + "logps/chosen": -410.6923828125, + "logps/rejected": -480.15234375, + "loss": 0.4415, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9327335357666016, + "rewards/margins": 1.0308973789215088, + "rewards/rejected": -2.9636306762695312, + "step": 3760 + }, + { + "epoch": 0.25, + "learning_rate": 4.67961641816062e-06, + "logits/chosen": -1.1115623712539673, + "logits/rejected": -0.9650887250900269, + "logps/chosen": -551.06494140625, + "logps/rejected": -606.406982421875, + "loss": 0.5092, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.7526817321777344, + "rewards/margins": 0.9667217135429382, + "rewards/rejected": -3.7194037437438965, + "step": 3770 + }, + { + "epoch": 0.25, + "learning_rate": 4.676814140906188e-06, + "logits/chosen": -0.6974936723709106, + "logits/rejected": -0.6922891736030579, + "logps/chosen": -617.8670654296875, + "logps/rejected": -644.30029296875, + "loss": 0.6464, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -3.783308506011963, + "rewards/margins": 0.5703891515731812, + "rewards/rejected": -4.353697776794434, + "step": 3780 + }, + { + "epoch": 0.25, + "learning_rate": 4.674000508322872e-06, + "logits/chosen": -0.44656461477279663, + "logits/rejected": -0.5972496867179871, + "logps/chosen": -527.9207153320312, + "logps/rejected": -598.678955078125, + "loss": 0.5704, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -3.10385799407959, + "rewards/margins": 0.6157468557357788, + "rewards/rejected": -3.719604969024658, + "step": 3790 + }, + { + "epoch": 0.25, + "learning_rate": 4.671175535087959e-06, + "logits/chosen": -0.6944919228553772, + "logits/rejected": -0.7936210632324219, + "logps/chosen": -692.2608642578125, + "logps/rejected": -807.3065185546875, + "loss": 0.4123, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -4.009331703186035, + "rewards/margins": 1.1827731132507324, + "rewards/rejected": -5.192105293273926, + "step": 3800 + }, + { + "epoch": 0.25, + "eval_logits/chosen": -0.6038689613342285, + "eval_logits/rejected": -0.5056295990943909, + "eval_logps/chosen": -590.8090209960938, + "eval_logps/rejected": -636.2662353515625, + "eval_loss": 0.5433777570724487, + "eval_rewards/accuracies": 0.6359999775886536, + "eval_rewards/chosen": -3.588040828704834, + "eval_rewards/margins": 0.6585022211074829, + "eval_rewards/rejected": -4.246542930603027, + "eval_runtime": 732.5781, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 3800 + }, + { + "epoch": 0.25, + "learning_rate": 4.6683392359378924e-06, + "logits/chosen": -0.5370118021965027, + "logits/rejected": -0.40376004576683044, + "logps/chosen": -573.9254150390625, + "logps/rejected": -630.9413452148438, + "loss": 0.5425, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -3.3877639770507812, + "rewards/margins": 0.825324535369873, + "rewards/rejected": -4.213088512420654, + "step": 3810 + }, + { + "epoch": 0.25, + "learning_rate": 4.665491625668198e-06, + "logits/chosen": -0.3310016691684723, + "logits/rejected": -0.38532063364982605, + "logps/chosen": -552.151611328125, + "logps/rejected": -631.1104736328125, + "loss": 0.6136, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.95904278755188, + "rewards/margins": 0.5540381669998169, + "rewards/rejected": -4.5130815505981445, + "step": 3820 + }, + { + "epoch": 0.25, + "learning_rate": 4.662632719133407e-06, + "logits/chosen": -0.5547534823417664, + "logits/rejected": -0.3702434301376343, + "logps/chosen": -486.48529052734375, + "logps/rejected": -477.41943359375, + "loss": 0.5439, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.556877374649048, + "rewards/margins": 0.5794159770011902, + "rewards/rejected": -3.136293411254883, + "step": 3830 + }, + { + "epoch": 0.25, + "learning_rate": 4.659762531246974e-06, + "logits/chosen": -0.3453252911567688, + "logits/rejected": -0.4048599302768707, + "logps/chosen": -570.9554443359375, + "logps/rejected": -577.3557739257812, + "loss": 0.6338, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -3.563767194747925, + "rewards/margins": 0.4035702645778656, + "rewards/rejected": -3.9673373699188232, + "step": 3840 + }, + { + "epoch": 0.25, + "learning_rate": 4.656881076981207e-06, + "logits/chosen": -0.7550775408744812, + "logits/rejected": -0.686637282371521, + "logps/chosen": -476.0919494628906, + "logps/rejected": -483.2123107910156, + "loss": 0.6476, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.6290855407714844, + "rewards/margins": 0.250190794467926, + "rewards/rejected": -2.8792760372161865, + "step": 3850 + }, + { + "epoch": 0.25, + "learning_rate": 4.653988371367183e-06, + "logits/chosen": -0.7826381921768188, + "logits/rejected": -0.47376060485839844, + "logps/chosen": -506.4967346191406, + "logps/rejected": -497.3008728027344, + "loss": 0.5432, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.6569275856018066, + "rewards/margins": 0.5201212167739868, + "rewards/rejected": -3.177048444747925, + "step": 3860 + }, + { + "epoch": 0.25, + "learning_rate": 4.651084429494671e-06, + "logits/chosen": -0.9120081663131714, + "logits/rejected": -0.6307070255279541, + "logps/chosen": -532.2633056640625, + "logps/rejected": -535.4483642578125, + "loss": 0.4548, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.569122791290283, + "rewards/margins": 0.8270888328552246, + "rewards/rejected": -3.3962111473083496, + "step": 3870 + }, + { + "epoch": 0.25, + "learning_rate": 4.648169266512053e-06, + "logits/chosen": -1.1297684907913208, + "logits/rejected": -0.8716074824333191, + "logps/chosen": -457.02001953125, + "logps/rejected": -477.3675231933594, + "loss": 0.5569, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3169193267822266, + "rewards/margins": 0.653015673160553, + "rewards/rejected": -2.969935178756714, + "step": 3880 + }, + { + "epoch": 0.25, + "learning_rate": 4.6452428976262505e-06, + "logits/chosen": -0.7666735053062439, + "logits/rejected": -0.477083683013916, + "logps/chosen": -437.20770263671875, + "logps/rejected": -507.29058837890625, + "loss": 0.3827, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3341622352600098, + "rewards/margins": 1.1440565586090088, + "rewards/rejected": -3.4782187938690186, + "step": 3890 + }, + { + "epoch": 0.26, + "learning_rate": 4.642305338102633e-06, + "logits/chosen": -0.6205832362174988, + "logits/rejected": -0.8550397157669067, + "logps/chosen": -445.5555725097656, + "logps/rejected": -522.7137451171875, + "loss": 0.6359, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.838059902191162, + "rewards/margins": 0.5901249647140503, + "rewards/rejected": -3.428185224533081, + "step": 3900 + }, + { + "epoch": 0.26, + "eval_logits/chosen": -0.794431209564209, + "eval_logits/rejected": -0.6801789402961731, + "eval_logps/chosen": -498.5151672363281, + "eval_logps/rejected": -564.9202880859375, + "eval_loss": 0.52691650390625, + "eval_rewards/accuracies": 0.640999972820282, + "eval_rewards/chosen": -2.665102243423462, + "eval_rewards/margins": 0.8679810762405396, + "eval_rewards/rejected": -3.533083438873291, + "eval_runtime": 734.9734, + "eval_samples_per_second": 2.721, + "eval_steps_per_second": 1.361, + "step": 3900 + }, + { + "epoch": 0.26, + "learning_rate": 4.639356603264953e-06, + "logits/chosen": -0.9367401003837585, + "logits/rejected": -0.9064401388168335, + "logps/chosen": -460.2066345214844, + "logps/rejected": -513.5260009765625, + "loss": 0.5192, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.183748483657837, + "rewards/margins": 0.7905017137527466, + "rewards/rejected": -2.974250078201294, + "step": 3910 + }, + { + "epoch": 0.26, + "learning_rate": 4.636396708495255e-06, + "logits/chosen": -0.751909613609314, + "logits/rejected": -0.7050498127937317, + "logps/chosen": -429.04901123046875, + "logps/rejected": -495.9883728027344, + "loss": 0.4816, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9985395669937134, + "rewards/margins": 0.9178323745727539, + "rewards/rejected": -2.916372060775757, + "step": 3920 + }, + { + "epoch": 0.26, + "learning_rate": 4.633425669233799e-06, + "logits/chosen": -1.1628475189208984, + "logits/rejected": -1.059190034866333, + "logps/chosen": -467.9149475097656, + "logps/rejected": -539.2251586914062, + "loss": 0.5287, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.330883264541626, + "rewards/margins": 0.7575754523277283, + "rewards/rejected": -3.08845853805542, + "step": 3930 + }, + { + "epoch": 0.26, + "learning_rate": 4.6304435009789825e-06, + "logits/chosen": -1.1164617538452148, + "logits/rejected": -0.8136693835258484, + "logps/chosen": -449.53515625, + "logps/rejected": -475.35546875, + "loss": 0.4778, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.12027907371521, + "rewards/margins": 0.9673018455505371, + "rewards/rejected": -3.087581157684326, + "step": 3940 + }, + { + "epoch": 0.26, + "learning_rate": 4.627450219287256e-06, + "logits/chosen": -1.1186714172363281, + "logits/rejected": -1.0646129846572876, + "logps/chosen": -405.68780517578125, + "logps/rejected": -469.1590270996094, + "loss": 0.5183, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.2412867546081543, + "rewards/margins": 0.8598229289054871, + "rewards/rejected": -3.101109743118286, + "step": 3950 + }, + { + "epoch": 0.26, + "learning_rate": 4.624445839773042e-06, + "logits/chosen": -0.9542514085769653, + "logits/rejected": -0.9420453906059265, + "logps/chosen": -403.17547607421875, + "logps/rejected": -441.06231689453125, + "loss": 0.6339, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.331641674041748, + "rewards/margins": 0.38950642943382263, + "rewards/rejected": -2.7211480140686035, + "step": 3960 + }, + { + "epoch": 0.26, + "learning_rate": 4.621430378108656e-06, + "logits/chosen": -1.0336421728134155, + "logits/rejected": -0.821884036064148, + "logps/chosen": -553.2286987304688, + "logps/rejected": -666.0933837890625, + "loss": 0.4842, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.9547042846679688, + "rewards/margins": 1.1962896585464478, + "rewards/rejected": -4.150994300842285, + "step": 3970 + }, + { + "epoch": 0.26, + "learning_rate": 4.618403850024223e-06, + "logits/chosen": -0.7831366658210754, + "logits/rejected": -0.7036348581314087, + "logps/chosen": -515.8258056640625, + "logps/rejected": -521.7974853515625, + "loss": 0.6226, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.603356122970581, + "rewards/margins": 0.5263395309448242, + "rewards/rejected": -3.129695415496826, + "step": 3980 + }, + { + "epoch": 0.26, + "learning_rate": 4.615366271307598e-06, + "logits/chosen": -1.0155903100967407, + "logits/rejected": -0.8377137184143066, + "logps/chosen": -435.54388427734375, + "logps/rejected": -492.73297119140625, + "loss": 0.5152, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.436009168624878, + "rewards/margins": 0.6908141374588013, + "rewards/rejected": -3.126823663711548, + "step": 3990 + }, + { + "epoch": 0.26, + "learning_rate": 4.612317657804277e-06, + "logits/chosen": -1.012681007385254, + "logits/rejected": -0.9964278340339661, + "logps/chosen": -388.09417724609375, + "logps/rejected": -510.4710388183594, + "loss": 0.5634, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.408611536026001, + "rewards/margins": 0.7102895975112915, + "rewards/rejected": -3.118901014328003, + "step": 4000 + }, + { + "epoch": 0.26, + "eval_logits/chosen": -1.0345293283462524, + "eval_logits/rejected": -0.9063413143157959, + "eval_logps/chosen": -468.72064208984375, + "eval_logps/rejected": -528.831298828125, + "eval_loss": 0.5224303007125854, + "eval_rewards/accuracies": 0.6514999866485596, + "eval_rewards/chosen": -2.367156744003296, + "eval_rewards/margins": 0.8050370812416077, + "eval_rewards/rejected": -3.172193765640259, + "eval_runtime": 735.7061, + "eval_samples_per_second": 2.718, + "eval_steps_per_second": 1.359, + "step": 4000 + }, + { + "epoch": 0.26, + "learning_rate": 4.6092580254173236e-06, + "logits/chosen": -0.9374804496765137, + "logits/rejected": -0.8715485334396362, + "logps/chosen": -493.97869873046875, + "logps/rejected": -579.1528930664062, + "loss": 0.4506, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.3665592670440674, + "rewards/margins": 1.0384232997894287, + "rewards/rejected": -3.404982805252075, + "step": 4010 + }, + { + "epoch": 0.26, + "learning_rate": 4.606187390107277e-06, + "logits/chosen": -0.930155873298645, + "logits/rejected": -0.8629436492919922, + "logps/chosen": -480.35479736328125, + "logps/rejected": -514.3657836914062, + "loss": 0.5612, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.554793119430542, + "rewards/margins": 0.7300376296043396, + "rewards/rejected": -3.2848312854766846, + "step": 4020 + }, + { + "epoch": 0.26, + "learning_rate": 4.603105767892077e-06, + "logits/chosen": -1.12532639503479, + "logits/rejected": -1.0434709787368774, + "logps/chosen": -453.9189453125, + "logps/rejected": -564.5955200195312, + "loss": 0.4922, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.5922884941101074, + "rewards/margins": 0.9272336959838867, + "rewards/rejected": -3.5195224285125732, + "step": 4030 + }, + { + "epoch": 0.26, + "learning_rate": 4.6000131748469725e-06, + "logits/chosen": -1.0651403665542603, + "logits/rejected": -0.9068595170974731, + "logps/chosen": -488.0533752441406, + "logps/rejected": -489.08306884765625, + "loss": 0.557, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.3791310787200928, + "rewards/margins": 0.7534803748130798, + "rewards/rejected": -3.1326117515563965, + "step": 4040 + }, + { + "epoch": 0.26, + "learning_rate": 4.596909627104445e-06, + "logits/chosen": -1.3376210927963257, + "logits/rejected": -1.189888596534729, + "logps/chosen": -528.3531494140625, + "logps/rejected": -602.3948974609375, + "loss": 0.5493, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.7961485385894775, + "rewards/margins": 1.0667587518692017, + "rewards/rejected": -3.8629074096679688, + "step": 4050 + }, + { + "epoch": 0.27, + "learning_rate": 4.5937951408541215e-06, + "logits/chosen": -1.3444876670837402, + "logits/rejected": -0.9238430857658386, + "logps/chosen": -442.65191650390625, + "logps/rejected": -513.8806762695312, + "loss": 0.4464, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9001344442367554, + "rewards/margins": 1.1660583019256592, + "rewards/rejected": -3.066192865371704, + "step": 4060 + }, + { + "epoch": 0.27, + "learning_rate": 4.590669732342685e-06, + "logits/chosen": -1.1866027116775513, + "logits/rejected": -1.0306710004806519, + "logps/chosen": -351.868896484375, + "logps/rejected": -462.2666015625, + "loss": 0.4385, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3972125053405762, + "rewards/margins": 1.085050344467163, + "rewards/rejected": -2.4822630882263184, + "step": 4070 + }, + { + "epoch": 0.27, + "learning_rate": 4.587533417873799e-06, + "logits/chosen": -1.2526047229766846, + "logits/rejected": -1.2084218263626099, + "logps/chosen": -376.30963134765625, + "logps/rejected": -536.0862426757812, + "loss": 0.508, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.831455945968628, + "rewards/margins": 1.0007293224334717, + "rewards/rejected": -2.832185745239258, + "step": 4080 + }, + { + "epoch": 0.27, + "learning_rate": 4.584386213808016e-06, + "logits/chosen": -1.204489827156067, + "logits/rejected": -1.0284204483032227, + "logps/chosen": -381.1136779785156, + "logps/rejected": -417.54241943359375, + "loss": 0.4861, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.594346523284912, + "rewards/margins": 0.8220494985580444, + "rewards/rejected": -2.416395902633667, + "step": 4090 + }, + { + "epoch": 0.27, + "learning_rate": 4.581228136562693e-06, + "logits/chosen": -1.2890905141830444, + "logits/rejected": -1.2788188457489014, + "logps/chosen": -375.24700927734375, + "logps/rejected": -383.44696044921875, + "loss": 0.7537, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.370665192604065, + "rewards/margins": 0.33884650468826294, + "rewards/rejected": -1.7095117568969727, + "step": 4100 + }, + { + "epoch": 0.27, + "eval_logits/chosen": -1.455415964126587, + "eval_logits/rejected": -1.3053381443023682, + "eval_logps/chosen": -354.7429504394531, + "eval_logps/rejected": -415.7259826660156, + "eval_loss": 0.5228714346885681, + "eval_rewards/accuracies": 0.6524999737739563, + "eval_rewards/chosen": -1.2273799180984497, + "eval_rewards/margins": 0.8137608766555786, + "eval_rewards/rejected": -2.041140556335449, + "eval_runtime": 733.1087, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 4100 + }, + { + "epoch": 0.27, + "learning_rate": 4.578059202611909e-06, + "logits/chosen": -1.5341708660125732, + "logits/rejected": -1.3989231586456299, + "logps/chosen": -353.5614013671875, + "logps/rejected": -389.3527526855469, + "loss": 0.5833, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.9640250205993652, + "rewards/margins": 0.5054730772972107, + "rewards/rejected": -1.4694980382919312, + "step": 4110 + }, + { + "epoch": 0.27, + "learning_rate": 4.574879428486376e-06, + "logits/chosen": -1.5479503870010376, + "logits/rejected": -1.4209856986999512, + "logps/chosen": -326.52545166015625, + "logps/rejected": -405.92315673828125, + "loss": 0.4775, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.1360893249511719, + "rewards/margins": 0.8231987953186035, + "rewards/rejected": -1.9592880010604858, + "step": 4120 + }, + { + "epoch": 0.27, + "learning_rate": 4.571688830773352e-06, + "logits/chosen": -1.571954607963562, + "logits/rejected": -1.4492496252059937, + "logps/chosen": -316.35675048828125, + "logps/rejected": -350.2168884277344, + "loss": 0.5704, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.9428855776786804, + "rewards/margins": 0.5373916625976562, + "rewards/rejected": -1.4802772998809814, + "step": 4130 + }, + { + "epoch": 0.27, + "learning_rate": 4.568487426116559e-06, + "logits/chosen": -1.3130545616149902, + "logits/rejected": -1.2582907676696777, + "logps/chosen": -284.15936279296875, + "logps/rejected": -345.6742248535156, + "loss": 0.495, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.1218750476837158, + "rewards/margins": 0.6904827356338501, + "rewards/rejected": -1.8123579025268555, + "step": 4140 + }, + { + "epoch": 0.27, + "learning_rate": 4.565275231216092e-06, + "logits/chosen": -0.9784256815910339, + "logits/rejected": -0.9782289266586304, + "logps/chosen": -308.6110534667969, + "logps/rejected": -411.9737243652344, + "loss": 0.5705, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.5726760625839233, + "rewards/margins": 0.5737274289131165, + "rewards/rejected": -2.1464033126831055, + "step": 4150 + }, + { + "epoch": 0.27, + "learning_rate": 4.562052262828331e-06, + "logits/chosen": -1.1103951930999756, + "logits/rejected": -1.0181940793991089, + "logps/chosen": -407.5032958984375, + "logps/rejected": -475.49053955078125, + "loss": 0.5093, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0869228839874268, + "rewards/margins": 0.7326322793960571, + "rewards/rejected": -2.819554567337036, + "step": 4160 + }, + { + "epoch": 0.27, + "learning_rate": 4.558818537765861e-06, + "logits/chosen": -1.2901301383972168, + "logits/rejected": -0.8406227231025696, + "logps/chosen": -489.4602966308594, + "logps/rejected": -529.7099609375, + "loss": 0.5104, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.534459114074707, + "rewards/margins": 0.7511767148971558, + "rewards/rejected": -3.2856357097625732, + "step": 4170 + }, + { + "epoch": 0.27, + "learning_rate": 4.555574072897374e-06, + "logits/chosen": -1.0267504453659058, + "logits/rejected": -1.101967215538025, + "logps/chosen": -465.4263610839844, + "logps/rejected": -522.2689819335938, + "loss": 0.5988, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.634650230407715, + "rewards/margins": 0.5911431312561035, + "rewards/rejected": -3.2257933616638184, + "step": 4180 + }, + { + "epoch": 0.27, + "learning_rate": 4.552318885147589e-06, + "logits/chosen": -1.147258996963501, + "logits/rejected": -0.8654224276542664, + "logps/chosen": -512.3572998046875, + "logps/rejected": -526.5074462890625, + "loss": 0.5374, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.714397430419922, + "rewards/margins": 0.7353962659835815, + "rewards/rejected": -3.449793577194214, + "step": 4190 + }, + { + "epoch": 0.27, + "learning_rate": 4.549052991497159e-06, + "logits/chosen": -1.0763828754425049, + "logits/rejected": -1.0327566862106323, + "logps/chosen": -412.31451416015625, + "logps/rejected": -481.2054748535156, + "loss": 0.5164, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3214733600616455, + "rewards/margins": 0.6847215890884399, + "rewards/rejected": -3.006194829940796, + "step": 4200 + }, + { + "epoch": 0.27, + "eval_logits/chosen": -1.0663347244262695, + "eval_logits/rejected": -0.9360623359680176, + "eval_logps/chosen": -458.21832275390625, + "eval_logps/rejected": -521.7139892578125, + "eval_loss": 0.5160852074623108, + "eval_rewards/accuracies": 0.6489999890327454, + "eval_rewards/chosen": -2.2621333599090576, + "eval_rewards/margins": 0.8388875722885132, + "eval_rewards/rejected": -3.1010210514068604, + "eval_runtime": 732.0631, + "eval_samples_per_second": 2.732, + "eval_steps_per_second": 1.366, + "step": 4200 + }, + { + "epoch": 0.28, + "learning_rate": 4.545776408982585e-06, + "logits/chosen": -1.1501169204711914, + "logits/rejected": -1.0862586498260498, + "logps/chosen": -438.3699645996094, + "logps/rejected": -511.65948486328125, + "loss": 0.5717, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0662589073181152, + "rewards/margins": 0.8230277895927429, + "rewards/rejected": -2.889286756515503, + "step": 4210 + }, + { + "epoch": 0.28, + "learning_rate": 4.542489154696128e-06, + "logits/chosen": -1.4092345237731934, + "logits/rejected": -1.092861533164978, + "logps/chosen": -429.67022705078125, + "logps/rejected": -432.35528564453125, + "loss": 0.5422, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.6194988489151, + "rewards/margins": 0.6393571496009827, + "rewards/rejected": -2.2588562965393066, + "step": 4220 + }, + { + "epoch": 0.28, + "learning_rate": 4.5391912457857145e-06, + "logits/chosen": -1.3653011322021484, + "logits/rejected": -1.1926562786102295, + "logps/chosen": -447.32830810546875, + "logps/rejected": -471.3085021972656, + "loss": 0.5818, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8235870599746704, + "rewards/margins": 0.7108983993530273, + "rewards/rejected": -2.534485101699829, + "step": 4230 + }, + { + "epoch": 0.28, + "learning_rate": 4.535882699454854e-06, + "logits/chosen": -1.38186514377594, + "logits/rejected": -1.286101222038269, + "logps/chosen": -433.41796875, + "logps/rejected": -538.211669921875, + "loss": 0.5182, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6236693859100342, + "rewards/margins": 0.8313106298446655, + "rewards/rejected": -2.4549801349639893, + "step": 4240 + }, + { + "epoch": 0.28, + "learning_rate": 4.532563532962546e-06, + "logits/chosen": -1.4803118705749512, + "logits/rejected": -1.4818512201309204, + "logps/chosen": -385.42938232421875, + "logps/rejected": -472.23822021484375, + "loss": 0.6099, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9553512334823608, + "rewards/margins": 0.6521388292312622, + "rewards/rejected": -2.607490062713623, + "step": 4250 + }, + { + "epoch": 0.28, + "learning_rate": 4.529233763623187e-06, + "logits/chosen": -1.283656358718872, + "logits/rejected": -1.011871576309204, + "logps/chosen": -374.8547058105469, + "logps/rejected": -392.13287353515625, + "loss": 0.5419, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7275416851043701, + "rewards/margins": 0.6523784399032593, + "rewards/rejected": -2.379920244216919, + "step": 4260 + }, + { + "epoch": 0.28, + "learning_rate": 4.5258934088064854e-06, + "logits/chosen": -1.2089446783065796, + "logits/rejected": -0.9043632745742798, + "logps/chosen": -398.29693603515625, + "logps/rejected": -429.73468017578125, + "loss": 0.5448, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7831614017486572, + "rewards/margins": 0.8613526225090027, + "rewards/rejected": -2.6445140838623047, + "step": 4270 + }, + { + "epoch": 0.28, + "learning_rate": 4.522542485937369e-06, + "logits/chosen": -1.2879055738449097, + "logits/rejected": -1.0973680019378662, + "logps/chosen": -433.8949279785156, + "logps/rejected": -444.53936767578125, + "loss": 0.3823, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.4324241876602173, + "rewards/margins": 1.0728528499603271, + "rewards/rejected": -2.505277156829834, + "step": 4280 + }, + { + "epoch": 0.28, + "learning_rate": 4.519181012495892e-06, + "logits/chosen": -1.4716758728027344, + "logits/rejected": -1.2221190929412842, + "logps/chosen": -359.55841064453125, + "logps/rejected": -416.4119567871094, + "loss": 0.5422, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.173391342163086, + "rewards/margins": 0.8592764139175415, + "rewards/rejected": -2.032667875289917, + "step": 4290 + }, + { + "epoch": 0.28, + "learning_rate": 4.515809006017147e-06, + "logits/chosen": -1.356048583984375, + "logits/rejected": -1.1803017854690552, + "logps/chosen": -332.74713134765625, + "logps/rejected": -356.1148376464844, + "loss": 0.6486, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.9933754801750183, + "rewards/margins": 0.5887696743011475, + "rewards/rejected": -1.5821453332901, + "step": 4300 + }, + { + "epoch": 0.28, + "eval_logits/chosen": -1.4796526432037354, + "eval_logits/rejected": -1.3301215171813965, + "eval_logps/chosen": -309.6466979980469, + "eval_logps/rejected": -364.43499755859375, + "eval_loss": 0.5246568322181702, + "eval_rewards/accuracies": 0.6549999713897705, + "eval_rewards/chosen": -0.7764175534248352, + "eval_rewards/margins": 0.7518131732940674, + "eval_rewards/rejected": -1.5282306671142578, + "eval_runtime": 730.8262, + "eval_samples_per_second": 2.737, + "eval_steps_per_second": 1.368, + "step": 4300 + }, + { + "epoch": 0.28, + "learning_rate": 4.512426484091171e-06, + "logits/chosen": -1.6821578741073608, + "logits/rejected": -1.423906922340393, + "logps/chosen": -345.78094482421875, + "logps/rejected": -375.2535400390625, + "loss": 0.5051, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6819978952407837, + "rewards/margins": 0.657656192779541, + "rewards/rejected": -1.3396542072296143, + "step": 4310 + }, + { + "epoch": 0.28, + "learning_rate": 4.509033464362858e-06, + "logits/chosen": -1.397766351699829, + "logits/rejected": -1.3964592218399048, + "logps/chosen": -296.751220703125, + "logps/rejected": -381.7581481933594, + "loss": 0.559, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.5548534393310547, + "rewards/margins": 0.7077598571777344, + "rewards/rejected": -1.262613296508789, + "step": 4320 + }, + { + "epoch": 0.28, + "learning_rate": 4.505629964531857e-06, + "logits/chosen": -1.527769923210144, + "logits/rejected": -1.3648316860198975, + "logps/chosen": -304.9037170410156, + "logps/rejected": -363.18463134765625, + "loss": 0.4798, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.8169935345649719, + "rewards/margins": 0.8909414410591125, + "rewards/rejected": -1.7079349756240845, + "step": 4330 + }, + { + "epoch": 0.28, + "learning_rate": 4.502216002352492e-06, + "logits/chosen": -1.4147685766220093, + "logits/rejected": -1.2035995721817017, + "logps/chosen": -251.2743682861328, + "logps/rejected": -303.8065490722656, + "loss": 0.5584, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.8823345899581909, + "rewards/margins": 0.7194640040397644, + "rewards/rejected": -1.6017987728118896, + "step": 4340 + }, + { + "epoch": 0.28, + "learning_rate": 4.498791595633663e-06, + "logits/chosen": -1.1233395338058472, + "logits/rejected": -0.8900758624076843, + "logps/chosen": -366.68353271484375, + "logps/rejected": -360.0097351074219, + "loss": 0.5087, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.042992353439331, + "rewards/margins": 0.8052667379379272, + "rewards/rejected": -1.8482589721679688, + "step": 4350 + }, + { + "epoch": 0.29, + "learning_rate": 4.495356762238751e-06, + "logits/chosen": -1.4454867839813232, + "logits/rejected": -1.0401540994644165, + "logps/chosen": -375.6465759277344, + "logps/rejected": -380.46240234375, + "loss": 0.4262, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.9244867563247681, + "rewards/margins": 1.0226781368255615, + "rewards/rejected": -1.9471648931503296, + "step": 4360 + }, + { + "epoch": 0.29, + "learning_rate": 4.491911520085532e-06, + "logits/chosen": -0.9478855133056641, + "logits/rejected": -0.9101594090461731, + "logps/chosen": -300.29791259765625, + "logps/rejected": -386.93060302734375, + "loss": 0.4718, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.9989013671875, + "rewards/margins": 0.7586010694503784, + "rewards/rejected": -1.757502555847168, + "step": 4370 + }, + { + "epoch": 0.29, + "learning_rate": 4.488455887146075e-06, + "logits/chosen": -1.1051117181777954, + "logits/rejected": -0.9938226938247681, + "logps/chosen": -313.7795104980469, + "logps/rejected": -451.63238525390625, + "loss": 0.4554, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.4366739988327026, + "rewards/margins": 1.2468922138214111, + "rewards/rejected": -2.6835663318634033, + "step": 4380 + }, + { + "epoch": 0.29, + "learning_rate": 4.484989881446654e-06, + "logits/chosen": -1.171975016593933, + "logits/rejected": -1.1171002388000488, + "logps/chosen": -373.6925964355469, + "logps/rejected": -412.12152099609375, + "loss": 0.5907, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7121661901474, + "rewards/margins": 0.5637361407279968, + "rewards/rejected": -2.275902271270752, + "step": 4390 + }, + { + "epoch": 0.29, + "learning_rate": 4.481513521067654e-06, + "logits/chosen": -1.1522445678710938, + "logits/rejected": -1.0051096677780151, + "logps/chosen": -414.10693359375, + "logps/rejected": -494.4396057128906, + "loss": 0.4663, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.893323302268982, + "rewards/margins": 1.1574846506118774, + "rewards/rejected": -3.0508081912994385, + "step": 4400 + }, + { + "epoch": 0.29, + "eval_logits/chosen": -1.0872026681900024, + "eval_logits/rejected": -0.9511972069740295, + "eval_logps/chosen": -398.82080078125, + "eval_logps/rejected": -475.67913818359375, + "eval_loss": 0.5215036273002625, + "eval_rewards/accuracies": 0.6524999737739563, + "eval_rewards/chosen": -1.6681584119796753, + "eval_rewards/margins": 0.9725137948989868, + "eval_rewards/rejected": -2.640672206878662, + "eval_runtime": 733.2598, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 4400 + }, + { + "epoch": 0.29, + "learning_rate": 4.478026824143473e-06, + "logits/chosen": -1.1751139163970947, + "logits/rejected": -1.1155116558074951, + "logps/chosen": -443.25335693359375, + "logps/rejected": -497.99713134765625, + "loss": 0.5358, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7719179391860962, + "rewards/margins": 1.1131114959716797, + "rewards/rejected": -2.8850293159484863, + "step": 4410 + }, + { + "epoch": 0.29, + "learning_rate": 4.474529808862429e-06, + "logits/chosen": -0.9981487393379211, + "logits/rejected": -1.037527084350586, + "logps/chosen": -364.75, + "logps/rejected": -485.3653869628906, + "loss": 0.4831, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7592971324920654, + "rewards/margins": 1.0299116373062134, + "rewards/rejected": -2.7892086505889893, + "step": 4420 + }, + { + "epoch": 0.29, + "learning_rate": 4.471022493466669e-06, + "logits/chosen": -1.0802857875823975, + "logits/rejected": -0.8232300877571106, + "logps/chosen": -509.3580627441406, + "logps/rejected": -502.99908447265625, + "loss": 0.5391, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.0741891860961914, + "rewards/margins": 0.7065232396125793, + "rewards/rejected": -2.780712604522705, + "step": 4430 + }, + { + "epoch": 0.29, + "learning_rate": 4.467504896252066e-06, + "logits/chosen": -1.2374316453933716, + "logits/rejected": -1.1976972818374634, + "logps/chosen": -458.05401611328125, + "logps/rejected": -505.687744140625, + "loss": 0.5669, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.083527088165283, + "rewards/margins": 0.7637595534324646, + "rewards/rejected": -2.8472867012023926, + "step": 4440 + }, + { + "epoch": 0.29, + "learning_rate": 4.463977035568132e-06, + "logits/chosen": -0.9392198324203491, + "logits/rejected": -1.0262138843536377, + "logps/chosen": -443.80462646484375, + "logps/rejected": -533.6358642578125, + "loss": 0.6599, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.314218282699585, + "rewards/margins": 0.3812568783760071, + "rewards/rejected": -2.6954751014709473, + "step": 4450 + }, + { + "epoch": 0.29, + "learning_rate": 4.460438929817914e-06, + "logits/chosen": -0.880610466003418, + "logits/rejected": -0.651411235332489, + "logps/chosen": -464.05908203125, + "logps/rejected": -528.5072021484375, + "loss": 0.5399, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.576392889022827, + "rewards/margins": 0.6873677968978882, + "rewards/rejected": -3.263760805130005, + "step": 4460 + }, + { + "epoch": 0.29, + "learning_rate": 4.456890597457907e-06, + "logits/chosen": -0.8304548263549805, + "logits/rejected": -0.7910754084587097, + "logps/chosen": -479.0946350097656, + "logps/rejected": -580.2449340820312, + "loss": 0.4911, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.6695892810821533, + "rewards/margins": 0.829459011554718, + "rewards/rejected": -3.4990482330322266, + "step": 4470 + }, + { + "epoch": 0.29, + "learning_rate": 4.453332056997951e-06, + "logits/chosen": -0.7744184732437134, + "logits/rejected": -0.7223728895187378, + "logps/chosen": -383.31866455078125, + "logps/rejected": -492.0791931152344, + "loss": 0.3992, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0335917472839355, + "rewards/margins": 1.1308565139770508, + "rewards/rejected": -3.1644484996795654, + "step": 4480 + }, + { + "epoch": 0.29, + "learning_rate": 4.449763327001134e-06, + "logits/chosen": -0.9835674166679382, + "logits/rejected": -1.0188270807266235, + "logps/chosen": -452.4715270996094, + "logps/rejected": -548.7013549804688, + "loss": 0.6053, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.6370930671691895, + "rewards/margins": 0.6781438589096069, + "rewards/rejected": -3.315237045288086, + "step": 4490 + }, + { + "epoch": 0.29, + "learning_rate": 4.446184426083702e-06, + "logits/chosen": -1.0631225109100342, + "logits/rejected": -0.8816837072372437, + "logps/chosen": -470.7005920410156, + "logps/rejected": -597.3644409179688, + "loss": 0.5322, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.771361827850342, + "rewards/margins": 1.1829630136489868, + "rewards/rejected": -3.9543251991271973, + "step": 4500 + }, + { + "epoch": 0.29, + "eval_logits/chosen": -1.0830296277999878, + "eval_logits/rejected": -0.9450811743736267, + "eval_logps/chosen": -466.5963439941406, + "eval_logps/rejected": -540.9030151367188, + "eval_loss": 0.5166224837303162, + "eval_rewards/accuracies": 0.6485000252723694, + "eval_rewards/chosen": -2.345913887023926, + "eval_rewards/margins": 0.9469969868659973, + "eval_rewards/rejected": -3.292910575866699, + "eval_runtime": 733.2167, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 4500 + }, + { + "epoch": 0.3, + "learning_rate": 4.442595372914954e-06, + "logits/chosen": -1.1284606456756592, + "logits/rejected": -1.064886450767517, + "logps/chosen": -433.7189025878906, + "logps/rejected": -444.3348083496094, + "loss": 0.4277, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9673858880996704, + "rewards/margins": 0.9589015245437622, + "rewards/rejected": -2.9262874126434326, + "step": 4510 + }, + { + "epoch": 0.3, + "learning_rate": 4.43899618621715e-06, + "logits/chosen": -1.1971828937530518, + "logits/rejected": -0.9432374238967896, + "logps/chosen": -486.7794494628906, + "logps/rejected": -599.4856567382812, + "loss": 0.5682, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.3572559356689453, + "rewards/margins": 1.0950950384140015, + "rewards/rejected": -3.4523510932922363, + "step": 4520 + }, + { + "epoch": 0.3, + "learning_rate": 4.4353868847654105e-06, + "logits/chosen": -1.4479076862335205, + "logits/rejected": -1.1744765043258667, + "logps/chosen": -417.41510009765625, + "logps/rejected": -470.2010803222656, + "loss": 0.5878, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7155687808990479, + "rewards/margins": 0.8285356760025024, + "rewards/rejected": -2.5441043376922607, + "step": 4530 + }, + { + "epoch": 0.3, + "learning_rate": 4.43176748738762e-06, + "logits/chosen": -1.175443410873413, + "logits/rejected": -1.074643850326538, + "logps/chosen": -436.22784423828125, + "logps/rejected": -551.2193603515625, + "loss": 0.5606, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.0427985191345215, + "rewards/margins": 1.1060316562652588, + "rewards/rejected": -3.148829936981201, + "step": 4540 + }, + { + "epoch": 0.3, + "learning_rate": 4.4281380129643295e-06, + "logits/chosen": -1.1706737279891968, + "logits/rejected": -0.9897640347480774, + "logps/chosen": -417.036865234375, + "logps/rejected": -527.3284912109375, + "loss": 0.5259, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.872867226600647, + "rewards/margins": 1.2076570987701416, + "rewards/rejected": -3.08052396774292, + "step": 4550 + }, + { + "epoch": 0.3, + "learning_rate": 4.424498480428654e-06, + "logits/chosen": -1.2937458753585815, + "logits/rejected": -1.1463016271591187, + "logps/chosen": -421.75616455078125, + "logps/rejected": -444.4380798339844, + "loss": 0.6206, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.744842529296875, + "rewards/margins": 0.627072811126709, + "rewards/rejected": -2.371915340423584, + "step": 4560 + }, + { + "epoch": 0.3, + "learning_rate": 4.420848908766178e-06, + "logits/chosen": -1.4309735298156738, + "logits/rejected": -1.3544584512710571, + "logps/chosen": -376.6891174316406, + "logps/rejected": -450.12078857421875, + "loss": 0.6033, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.6946220397949219, + "rewards/margins": 0.6455153226852417, + "rewards/rejected": -2.340137243270874, + "step": 4570 + }, + { + "epoch": 0.3, + "learning_rate": 4.417189317014855e-06, + "logits/chosen": -1.3555351495742798, + "logits/rejected": -1.5094717741012573, + "logps/chosen": -344.8653564453125, + "logps/rejected": -409.7660217285156, + "loss": 0.7424, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.4406945705413818, + "rewards/margins": 0.34203729033470154, + "rewards/rejected": -1.7827320098876953, + "step": 4580 + }, + { + "epoch": 0.3, + "learning_rate": 4.41351972426491e-06, + "logits/chosen": -1.2469980716705322, + "logits/rejected": -1.2732794284820557, + "logps/chosen": -381.6789245605469, + "logps/rejected": -504.3853454589844, + "loss": 0.4827, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.3493956327438354, + "rewards/margins": 0.6865336298942566, + "rewards/rejected": -2.0359294414520264, + "step": 4590 + }, + { + "epoch": 0.3, + "learning_rate": 4.409840149658735e-06, + "logits/chosen": -1.3501039743423462, + "logits/rejected": -1.2072316408157349, + "logps/chosen": -397.30950927734375, + "logps/rejected": -402.70428466796875, + "loss": 0.5485, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.127922534942627, + "rewards/margins": 0.5554654002189636, + "rewards/rejected": -1.6833879947662354, + "step": 4600 + }, + { + "epoch": 0.3, + "eval_logits/chosen": -1.3868718147277832, + "eval_logits/rejected": -1.245092749595642, + "eval_logps/chosen": -361.0744323730469, + "eval_logps/rejected": -399.0142822265625, + "eval_loss": 0.5370956063270569, + "eval_rewards/accuracies": 0.6510000228881836, + "eval_rewards/chosen": -1.2906944751739502, + "eval_rewards/margins": 0.5833296775817871, + "eval_rewards/rejected": -1.8740240335464478, + "eval_runtime": 733.0152, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 4600 + }, + { + "epoch": 0.3, + "learning_rate": 4.4061506123907925e-06, + "logits/chosen": -1.3194220066070557, + "logits/rejected": -1.1732145547866821, + "logps/chosen": -410.449462890625, + "logps/rejected": -420.6064453125, + "loss": 0.5794, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.4635764360427856, + "rewards/margins": 0.504631519317627, + "rewards/rejected": -1.9682079553604126, + "step": 4610 + }, + { + "epoch": 0.3, + "learning_rate": 4.402451131707519e-06, + "logits/chosen": -1.479317545890808, + "logits/rejected": -1.1270101070404053, + "logps/chosen": -348.9317321777344, + "logps/rejected": -352.8699645996094, + "loss": 0.4994, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.394663691520691, + "rewards/margins": 0.794195830821991, + "rewards/rejected": -2.188859462738037, + "step": 4620 + }, + { + "epoch": 0.3, + "learning_rate": 4.398741726907215e-06, + "logits/chosen": -1.6055643558502197, + "logits/rejected": -1.2944947481155396, + "logps/chosen": -413.595947265625, + "logps/rejected": -456.23095703125, + "loss": 0.4513, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.343385934829712, + "rewards/margins": 0.829616367816925, + "rewards/rejected": -2.173002243041992, + "step": 4630 + }, + { + "epoch": 0.3, + "learning_rate": 4.395022417339955e-06, + "logits/chosen": -1.2120579481124878, + "logits/rejected": -1.1644296646118164, + "logps/chosen": -390.01263427734375, + "logps/rejected": -480.06280517578125, + "loss": 0.5869, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8338760137557983, + "rewards/margins": 0.8156105279922485, + "rewards/rejected": -2.649486541748047, + "step": 4640 + }, + { + "epoch": 0.3, + "learning_rate": 4.391293222407479e-06, + "logits/chosen": -1.1576309204101562, + "logits/rejected": -1.2002023458480835, + "logps/chosen": -268.47930908203125, + "logps/rejected": -357.7801208496094, + "loss": 0.5439, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.3192216157913208, + "rewards/margins": 0.71070796251297, + "rewards/rejected": -2.0299293994903564, + "step": 4650 + }, + { + "epoch": 0.3, + "learning_rate": 4.387554161563094e-06, + "logits/chosen": -1.2241219282150269, + "logits/rejected": -1.166908860206604, + "logps/chosen": -400.4576721191406, + "logps/rejected": -520.9844360351562, + "loss": 0.4558, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.015934467315674, + "rewards/margins": 1.3234636783599854, + "rewards/rejected": -3.3393986225128174, + "step": 4660 + }, + { + "epoch": 0.31, + "learning_rate": 4.383805254311575e-06, + "logits/chosen": -1.3214675188064575, + "logits/rejected": -0.9820463061332703, + "logps/chosen": -452.3216857910156, + "logps/rejected": -490.5860290527344, + "loss": 0.4993, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.961806058883667, + "rewards/margins": 0.8383065462112427, + "rewards/rejected": -2.80011248588562, + "step": 4670 + }, + { + "epoch": 0.31, + "learning_rate": 4.380046520209056e-06, + "logits/chosen": -1.1236032247543335, + "logits/rejected": -0.865643322467804, + "logps/chosen": -391.35614013671875, + "logps/rejected": -480.70025634765625, + "loss": 0.4831, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.910897970199585, + "rewards/margins": 1.1318318843841553, + "rewards/rejected": -3.0427298545837402, + "step": 4680 + }, + { + "epoch": 0.31, + "learning_rate": 4.376277978862936e-06, + "logits/chosen": -0.8476250767707825, + "logits/rejected": -0.7707692384719849, + "logps/chosen": -436.7411193847656, + "logps/rejected": -474.6356506347656, + "loss": 0.5271, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.1075243949890137, + "rewards/margins": 0.7842603921890259, + "rewards/rejected": -2.891785144805908, + "step": 4690 + }, + { + "epoch": 0.31, + "learning_rate": 4.372499649931774e-06, + "logits/chosen": -1.027212142944336, + "logits/rejected": -0.9219115972518921, + "logps/chosen": -472.2177734375, + "logps/rejected": -634.4398193359375, + "loss": 0.4012, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.6312623023986816, + "rewards/margins": 1.5426554679870605, + "rewards/rejected": -4.173917770385742, + "step": 4700 + }, + { + "epoch": 0.31, + "eval_logits/chosen": -0.963469386100769, + "eval_logits/rejected": -0.8301882147789001, + "eval_logps/chosen": -495.0129089355469, + "eval_logps/rejected": -579.796142578125, + "eval_loss": 0.518997848033905, + "eval_rewards/accuracies": 0.6514999866485596, + "eval_rewards/chosen": -2.630079984664917, + "eval_rewards/margins": 1.0517627000808716, + "eval_rewards/rejected": -3.68184232711792, + "eval_runtime": 734.3909, + "eval_samples_per_second": 2.723, + "eval_steps_per_second": 1.362, + "step": 4700 + }, + { + "epoch": 0.31, + "learning_rate": 4.368711553125185e-06, + "logits/chosen": -1.170462727546692, + "logits/rejected": -1.1017426252365112, + "logps/chosen": -520.1934814453125, + "logps/rejected": -549.4090576171875, + "loss": 0.5431, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.430034875869751, + "rewards/margins": 0.876590371131897, + "rewards/rejected": -3.3066253662109375, + "step": 4710 + }, + { + "epoch": 0.31, + "learning_rate": 4.364913708203734e-06, + "logits/chosen": -1.2170623540878296, + "logits/rejected": -1.0090713500976562, + "logps/chosen": -513.0048217773438, + "logps/rejected": -546.5401611328125, + "loss": 0.4402, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.28108549118042, + "rewards/margins": 1.0895252227783203, + "rewards/rejected": -3.3706107139587402, + "step": 4720 + }, + { + "epoch": 0.31, + "learning_rate": 4.361106134978844e-06, + "logits/chosen": -1.0908490419387817, + "logits/rejected": -0.8982577323913574, + "logps/chosen": -496.34521484375, + "logps/rejected": -563.3717041015625, + "loss": 0.5615, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.259147882461548, + "rewards/margins": 0.7997722029685974, + "rewards/rejected": -3.058920383453369, + "step": 4730 + }, + { + "epoch": 0.31, + "learning_rate": 4.357288853312681e-06, + "logits/chosen": -1.200878381729126, + "logits/rejected": -1.1836717128753662, + "logps/chosen": -469.6875915527344, + "logps/rejected": -538.845703125, + "loss": 0.5054, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8620973825454712, + "rewards/margins": 0.7389506101608276, + "rewards/rejected": -2.601047992706299, + "step": 4740 + }, + { + "epoch": 0.31, + "learning_rate": 4.353461883118056e-06, + "logits/chosen": -1.1654294729232788, + "logits/rejected": -1.0323411226272583, + "logps/chosen": -443.4126892089844, + "logps/rejected": -486.7259826660156, + "loss": 0.5934, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.141221284866333, + "rewards/margins": 0.6446442604064941, + "rewards/rejected": -2.785865306854248, + "step": 4750 + }, + { + "epoch": 0.31, + "learning_rate": 4.34962524435832e-06, + "logits/chosen": -1.076738953590393, + "logits/rejected": -0.9777164459228516, + "logps/chosen": -405.94659423828125, + "logps/rejected": -497.431396484375, + "loss": 0.4841, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.8626617193222046, + "rewards/margins": 1.283182144165039, + "rewards/rejected": -3.145843744277954, + "step": 4760 + }, + { + "epoch": 0.31, + "learning_rate": 4.34577895704726e-06, + "logits/chosen": -1.4177812337875366, + "logits/rejected": -1.2205036878585815, + "logps/chosen": -461.11138916015625, + "logps/rejected": -569.2667846679688, + "loss": 0.4604, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9949737787246704, + "rewards/margins": 1.3387939929962158, + "rewards/rejected": -3.3337676525115967, + "step": 4770 + }, + { + "epoch": 0.31, + "learning_rate": 4.3419230412489954e-06, + "logits/chosen": -1.3925700187683105, + "logits/rejected": -1.1407649517059326, + "logps/chosen": -505.38604736328125, + "logps/rejected": -519.0608520507812, + "loss": 0.5027, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.160182237625122, + "rewards/margins": 0.897386372089386, + "rewards/rejected": -3.0575685501098633, + "step": 4780 + }, + { + "epoch": 0.31, + "learning_rate": 4.338057517077872e-06, + "logits/chosen": -1.2423205375671387, + "logits/rejected": -1.012940764427185, + "logps/chosen": -383.737060546875, + "logps/rejected": -526.2789306640625, + "loss": 0.3283, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.896958589553833, + "rewards/margins": 1.8589407205581665, + "rewards/rejected": -3.755898952484131, + "step": 4790 + }, + { + "epoch": 0.31, + "learning_rate": 4.334182404698356e-06, + "logits/chosen": -1.266410231590271, + "logits/rejected": -0.9841598272323608, + "logps/chosen": -423.17169189453125, + "logps/rejected": -452.947265625, + "loss": 0.4963, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9306468963623047, + "rewards/margins": 1.0817534923553467, + "rewards/rejected": -3.0124001502990723, + "step": 4800 + }, + { + "epoch": 0.31, + "eval_logits/chosen": -1.153753638267517, + "eval_logits/rejected": -1.0117337703704834, + "eval_logps/chosen": -424.8492431640625, + "eval_logps/rejected": -512.7779541015625, + "eval_loss": 0.512640118598938, + "eval_rewards/accuracies": 0.6539999842643738, + "eval_rewards/chosen": -1.9284428358078003, + "eval_rewards/margins": 1.0832173824310303, + "eval_rewards/rejected": -3.01166033744812, + "eval_runtime": 734.3129, + "eval_samples_per_second": 2.724, + "eval_steps_per_second": 1.362, + "step": 4800 + }, + { + "epoch": 0.31, + "learning_rate": 4.330297724324933e-06, + "logits/chosen": -1.4888598918914795, + "logits/rejected": -1.028956651687622, + "logps/chosen": -497.24462890625, + "logps/rejected": -483.7391052246094, + "loss": 0.5424, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.8883317708969116, + "rewards/margins": 0.8769477605819702, + "rewards/rejected": -2.765279531478882, + "step": 4810 + }, + { + "epoch": 0.32, + "learning_rate": 4.326403496221999e-06, + "logits/chosen": -1.2531269788742065, + "logits/rejected": -1.2295105457305908, + "logps/chosen": -298.4060974121094, + "logps/rejected": -327.56494140625, + "loss": 0.6945, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.3697706460952759, + "rewards/margins": 0.5593878626823425, + "rewards/rejected": -1.9291585683822632, + "step": 4820 + }, + { + "epoch": 0.32, + "learning_rate": 4.322499740703755e-06, + "logits/chosen": -1.433562994003296, + "logits/rejected": -1.5193655490875244, + "logps/chosen": -271.50836181640625, + "logps/rejected": -365.73895263671875, + "loss": 0.4978, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.7940917015075684, + "rewards/margins": 0.6970726251602173, + "rewards/rejected": -1.4911643266677856, + "step": 4830 + }, + { + "epoch": 0.32, + "learning_rate": 4.318586478134101e-06, + "logits/chosen": -1.5995076894760132, + "logits/rejected": -1.447758674621582, + "logps/chosen": -257.02606201171875, + "logps/rejected": -259.845703125, + "loss": 0.6046, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.643585741519928, + "rewards/margins": 0.4334011971950531, + "rewards/rejected": -1.0769869089126587, + "step": 4840 + }, + { + "epoch": 0.32, + "learning_rate": 4.314663728926534e-06, + "logits/chosen": -1.8331336975097656, + "logits/rejected": -1.5674731731414795, + "logps/chosen": -335.2432556152344, + "logps/rejected": -360.13116455078125, + "loss": 0.6137, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.7836753129959106, + "rewards/margins": 0.35466259717941284, + "rewards/rejected": -1.1383378505706787, + "step": 4850 + }, + { + "epoch": 0.32, + "learning_rate": 4.310731513544033e-06, + "logits/chosen": -1.556740403175354, + "logits/rejected": -1.3384201526641846, + "logps/chosen": -325.8907165527344, + "logps/rejected": -362.38525390625, + "loss": 0.4867, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.8303295373916626, + "rewards/margins": 0.825741171836853, + "rewards/rejected": -1.6560704708099365, + "step": 4860 + }, + { + "epoch": 0.32, + "learning_rate": 4.30678985249896e-06, + "logits/chosen": -1.411836862564087, + "logits/rejected": -1.3843629360198975, + "logps/chosen": -257.8215637207031, + "logps/rejected": -395.88067626953125, + "loss": 0.3987, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.0067152976989746, + "rewards/margins": 1.1979559659957886, + "rewards/rejected": -2.2046713829040527, + "step": 4870 + }, + { + "epoch": 0.32, + "learning_rate": 4.302838766352952e-06, + "logits/chosen": -1.1780803203582764, + "logits/rejected": -1.026503324508667, + "logps/chosen": -452.2996520996094, + "logps/rejected": -511.9593811035156, + "loss": 0.4642, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.967066764831543, + "rewards/margins": 0.9545124173164368, + "rewards/rejected": -2.921579360961914, + "step": 4880 + }, + { + "epoch": 0.32, + "learning_rate": 4.298878275716806e-06, + "logits/chosen": -0.8606294393539429, + "logits/rejected": -0.8033844232559204, + "logps/chosen": -475.10150146484375, + "logps/rejected": -563.3385620117188, + "loss": 0.5716, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.8322205543518066, + "rewards/margins": 0.9103401303291321, + "rewards/rejected": -3.742560863494873, + "step": 4890 + }, + { + "epoch": 0.32, + "learning_rate": 4.294908401250386e-06, + "logits/chosen": -0.9198969006538391, + "logits/rejected": -0.7157684564590454, + "logps/chosen": -516.4268798828125, + "logps/rejected": -558.6986083984375, + "loss": 0.5004, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -3.016838312149048, + "rewards/margins": 0.8680410385131836, + "rewards/rejected": -3.8848788738250732, + "step": 4900 + }, + { + "epoch": 0.32, + "eval_logits/chosen": -0.8907901048660278, + "eval_logits/rejected": -0.7704294323921204, + "eval_logps/chosen": -526.6473388671875, + "eval_logps/rejected": -583.9198608398438, + "eval_loss": 0.5151007771492004, + "eval_rewards/accuracies": 0.6614999771118164, + "eval_rewards/chosen": -2.9464235305786133, + "eval_rewards/margins": 0.776655912399292, + "eval_rewards/rejected": -3.7230799198150635, + "eval_runtime": 732.5685, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 4900 + }, + { + "epoch": 0.32, + "learning_rate": 4.290929163662498e-06, + "logits/chosen": -0.6321516633033752, + "logits/rejected": -0.6172500848770142, + "logps/chosen": -548.6419677734375, + "logps/rejected": -562.7332153320312, + "loss": 0.5349, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.7890663146972656, + "rewards/margins": 0.7180782556533813, + "rewards/rejected": -3.5071444511413574, + "step": 4910 + }, + { + "epoch": 0.32, + "learning_rate": 4.286940583710796e-06, + "logits/chosen": -1.0743346214294434, + "logits/rejected": -0.8789170384407043, + "logps/chosen": -624.6532592773438, + "logps/rejected": -672.2891845703125, + "loss": 0.4332, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -3.3190460205078125, + "rewards/margins": 1.042657494544983, + "rewards/rejected": -4.361703872680664, + "step": 4920 + }, + { + "epoch": 0.32, + "learning_rate": 4.282942682201667e-06, + "logits/chosen": -0.9150835871696472, + "logits/rejected": -0.7225080728530884, + "logps/chosen": -558.1234741210938, + "logps/rejected": -603.1954956054688, + "loss": 0.5136, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -3.06015682220459, + "rewards/margins": 0.8465850949287415, + "rewards/rejected": -3.9067416191101074, + "step": 4930 + }, + { + "epoch": 0.32, + "learning_rate": 4.278935479990123e-06, + "logits/chosen": -1.1814619302749634, + "logits/rejected": -0.840973973274231, + "logps/chosen": -482.06982421875, + "logps/rejected": -508.01141357421875, + "loss": 0.5504, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.774064540863037, + "rewards/margins": 0.6891308426856995, + "rewards/rejected": -3.463195323944092, + "step": 4940 + }, + { + "epoch": 0.32, + "learning_rate": 4.274918997979695e-06, + "logits/chosen": -0.9456547498703003, + "logits/rejected": -0.9622589945793152, + "logps/chosen": -505.5489196777344, + "logps/rejected": -578.9075927734375, + "loss": 0.6068, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -3.1191391944885254, + "rewards/margins": 0.7162333130836487, + "rewards/rejected": -3.8353724479675293, + "step": 4950 + }, + { + "epoch": 0.32, + "learning_rate": 4.270893257122319e-06, + "logits/chosen": -0.8949559330940247, + "logits/rejected": -0.742863118648529, + "logps/chosen": -508.76806640625, + "logps/rejected": -651.4818115234375, + "loss": 0.4558, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.8004612922668457, + "rewards/margins": 1.1320440769195557, + "rewards/rejected": -3.9325053691864014, + "step": 4960 + }, + { + "epoch": 0.33, + "learning_rate": 4.266858278418232e-06, + "logits/chosen": -0.7636032104492188, + "logits/rejected": -0.7520279884338379, + "logps/chosen": -473.548583984375, + "logps/rejected": -536.2864379882812, + "loss": 0.5227, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.369443416595459, + "rewards/margins": 0.8308904767036438, + "rewards/rejected": -3.200334072113037, + "step": 4970 + }, + { + "epoch": 0.33, + "learning_rate": 4.26281408291586e-06, + "logits/chosen": -1.1534435749053955, + "logits/rejected": -0.9177444577217102, + "logps/chosen": -476.447021484375, + "logps/rejected": -567.8323974609375, + "loss": 0.4335, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.341097116470337, + "rewards/margins": 1.209177017211914, + "rewards/rejected": -3.55027437210083, + "step": 4980 + }, + { + "epoch": 0.33, + "learning_rate": 4.258760691711706e-06, + "logits/chosen": -1.1102027893066406, + "logits/rejected": -0.977056622505188, + "logps/chosen": -428.1145935058594, + "logps/rejected": -505.6068420410156, + "loss": 0.5251, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.306530475616455, + "rewards/margins": 0.8208501935005188, + "rewards/rejected": -3.12738037109375, + "step": 4990 + }, + { + "epoch": 0.33, + "learning_rate": 4.254698125950247e-06, + "logits/chosen": -1.346663475036621, + "logits/rejected": -1.1366852521896362, + "logps/chosen": -513.910400390625, + "logps/rejected": -552.2686767578125, + "loss": 0.465, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.1289587020874023, + "rewards/margins": 0.8502386808395386, + "rewards/rejected": -2.9791972637176514, + "step": 5000 + }, + { + "epoch": 0.33, + "eval_logits/chosen": -1.0639259815216064, + "eval_logits/rejected": -0.934317409992218, + "eval_logps/chosen": -465.99224853515625, + "eval_logps/rejected": -532.8919677734375, + "eval_loss": 0.5096431374549866, + "eval_rewards/accuracies": 0.6675000190734863, + "eval_rewards/chosen": -2.3398725986480713, + "eval_rewards/margins": 0.8729275465011597, + "eval_rewards/rejected": -3.2128000259399414, + "eval_runtime": 734.3385, + "eval_samples_per_second": 2.724, + "eval_steps_per_second": 1.362, + "step": 5000 + }, + { + "epoch": 0.33, + "learning_rate": 4.250626406823815e-06, + "logits/chosen": -1.1543405055999756, + "logits/rejected": -0.9085921049118042, + "logps/chosen": -472.118896484375, + "logps/rejected": -643.9099731445312, + "loss": 0.4521, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.5525622367858887, + "rewards/margins": 1.5561296939849854, + "rewards/rejected": -4.108692169189453, + "step": 5010 + }, + { + "epoch": 0.33, + "learning_rate": 4.246545555572489e-06, + "logits/chosen": -0.8793859481811523, + "logits/rejected": -0.8848443031311035, + "logps/chosen": -409.1986999511719, + "logps/rejected": -546.0990600585938, + "loss": 0.5188, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.550025463104248, + "rewards/margins": 1.1194829940795898, + "rewards/rejected": -3.669508457183838, + "step": 5020 + }, + { + "epoch": 0.33, + "learning_rate": 4.242455593483992e-06, + "logits/chosen": -1.1510926485061646, + "logits/rejected": -0.9685741662979126, + "logps/chosen": -441.26348876953125, + "logps/rejected": -480.549560546875, + "loss": 0.4735, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.2210960388183594, + "rewards/margins": 0.8899548649787903, + "rewards/rejected": -3.111051082611084, + "step": 5030 + }, + { + "epoch": 0.33, + "learning_rate": 4.238356541893567e-06, + "logits/chosen": -1.0632343292236328, + "logits/rejected": -1.0304065942764282, + "logps/chosen": -393.16845703125, + "logps/rejected": -484.4117736816406, + "loss": 0.5064, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.03035306930542, + "rewards/margins": 1.0678585767745972, + "rewards/rejected": -3.0982117652893066, + "step": 5040 + }, + { + "epoch": 0.33, + "learning_rate": 4.234248422183876e-06, + "logits/chosen": -1.16147780418396, + "logits/rejected": -1.3117892742156982, + "logps/chosen": -407.19354248046875, + "logps/rejected": -506.6943359375, + "loss": 0.4877, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.6425886154174805, + "rewards/margins": 1.0405709743499756, + "rewards/rejected": -2.683159589767456, + "step": 5050 + }, + { + "epoch": 0.33, + "learning_rate": 4.230131255784884e-06, + "logits/chosen": -1.5039904117584229, + "logits/rejected": -1.2198777198791504, + "logps/chosen": -476.0791015625, + "logps/rejected": -579.59375, + "loss": 0.5034, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.181959629058838, + "rewards/margins": 1.2231109142303467, + "rewards/rejected": -3.4050700664520264, + "step": 5060 + }, + { + "epoch": 0.33, + "learning_rate": 4.226005064173748e-06, + "logits/chosen": -1.1899694204330444, + "logits/rejected": -1.0733693838119507, + "logps/chosen": -484.65301513671875, + "logps/rejected": -579.5365600585938, + "loss": 0.5832, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.200087785720825, + "rewards/margins": 0.7642645239830017, + "rewards/rejected": -2.9643523693084717, + "step": 5070 + }, + { + "epoch": 0.33, + "learning_rate": 4.2218698688747035e-06, + "logits/chosen": -0.830842137336731, + "logits/rejected": -0.6599709987640381, + "logps/chosen": -473.8846130371094, + "logps/rejected": -529.7459716796875, + "loss": 0.5309, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.411752939224243, + "rewards/margins": 1.0099151134490967, + "rewards/rejected": -3.4216678142547607, + "step": 5080 + }, + { + "epoch": 0.33, + "learning_rate": 4.217725691458957e-06, + "logits/chosen": -1.3030509948730469, + "logits/rejected": -1.1121243238449097, + "logps/chosen": -401.26605224609375, + "logps/rejected": -516.9102783203125, + "loss": 0.5194, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1704189777374268, + "rewards/margins": 0.8869827389717102, + "rewards/rejected": -3.057401418685913, + "step": 5090 + }, + { + "epoch": 0.33, + "learning_rate": 4.213572553544565e-06, + "logits/chosen": -0.968209445476532, + "logits/rejected": -0.9317210912704468, + "logps/chosen": -464.62896728515625, + "logps/rejected": -547.8236083984375, + "loss": 0.4609, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.2365708351135254, + "rewards/margins": 0.9055919647216797, + "rewards/rejected": -3.142162799835205, + "step": 5100 + }, + { + "epoch": 0.33, + "eval_logits/chosen": -1.0512932538986206, + "eval_logits/rejected": -0.9174529910087585, + "eval_logps/chosen": -430.6408996582031, + "eval_logps/rejected": -500.292236328125, + "eval_loss": 0.5073493123054504, + "eval_rewards/accuracies": 0.6654999852180481, + "eval_rewards/chosen": -1.9863591194152832, + "eval_rewards/margins": 0.900443971157074, + "eval_rewards/rejected": -2.886803388595581, + "eval_runtime": 733.5989, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 5100 + }, + { + "epoch": 0.33, + "learning_rate": 4.209410476796331e-06, + "logits/chosen": -0.9567044973373413, + "logits/rejected": -0.9498974084854126, + "logps/chosen": -364.0614318847656, + "logps/rejected": -427.20098876953125, + "loss": 0.5464, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9297001361846924, + "rewards/margins": 0.7312873005867004, + "rewards/rejected": -2.660987377166748, + "step": 5110 + }, + { + "epoch": 0.33, + "learning_rate": 4.205239482925686e-06, + "logits/chosen": -0.9890548586845398, + "logits/rejected": -0.9276937246322632, + "logps/chosen": -364.5030212402344, + "logps/rejected": -465.77105712890625, + "loss": 0.5325, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7668765783309937, + "rewards/margins": 0.7914160490036011, + "rewards/rejected": -2.558292865753174, + "step": 5120 + }, + { + "epoch": 0.34, + "learning_rate": 4.201059593690577e-06, + "logits/chosen": -1.2910683155059814, + "logits/rejected": -1.229804277420044, + "logps/chosen": -417.73046875, + "logps/rejected": -458.9871520996094, + "loss": 0.5824, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9214271306991577, + "rewards/margins": 0.6668572425842285, + "rewards/rejected": -2.588284492492676, + "step": 5130 + }, + { + "epoch": 0.34, + "learning_rate": 4.196870830895354e-06, + "logits/chosen": -1.0787732601165771, + "logits/rejected": -0.9965429306030273, + "logps/chosen": -433.9598693847656, + "logps/rejected": -561.2127685546875, + "loss": 0.5479, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7621228694915771, + "rewards/margins": 0.7774554491043091, + "rewards/rejected": -2.5395781993865967, + "step": 5140 + }, + { + "epoch": 0.34, + "learning_rate": 4.192673216390657e-06, + "logits/chosen": -1.1716907024383545, + "logits/rejected": -0.9472560882568359, + "logps/chosen": -420.86053466796875, + "logps/rejected": -453.2992248535156, + "loss": 0.5648, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8788764476776123, + "rewards/margins": 0.7076621055603027, + "rewards/rejected": -2.586538553237915, + "step": 5150 + }, + { + "epoch": 0.34, + "learning_rate": 4.188466772073296e-06, + "logits/chosen": -1.2703173160552979, + "logits/rejected": -1.1680748462677002, + "logps/chosen": -376.0206298828125, + "logps/rejected": -413.83392333984375, + "loss": 0.5771, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.55496084690094, + "rewards/margins": 0.5857495069503784, + "rewards/rejected": -2.1407103538513184, + "step": 5160 + }, + { + "epoch": 0.34, + "learning_rate": 4.184251519886148e-06, + "logits/chosen": -1.160733938217163, + "logits/rejected": -1.102087378501892, + "logps/chosen": -395.3398132324219, + "logps/rejected": -477.61346435546875, + "loss": 0.612, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9923226833343506, + "rewards/margins": 0.5361863970756531, + "rewards/rejected": -2.5285091400146484, + "step": 5170 + }, + { + "epoch": 0.34, + "learning_rate": 4.180027481818033e-06, + "logits/chosen": -1.25356125831604, + "logits/rejected": -1.3256597518920898, + "logps/chosen": -432.5494079589844, + "logps/rejected": -464.7425231933594, + "loss": 0.5836, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.6585134267807007, + "rewards/margins": 0.7002665996551514, + "rewards/rejected": -2.3587799072265625, + "step": 5180 + }, + { + "epoch": 0.34, + "learning_rate": 4.175794679903602e-06, + "logits/chosen": -1.1973209381103516, + "logits/rejected": -1.0242760181427002, + "logps/chosen": -373.4833984375, + "logps/rejected": -385.53668212890625, + "loss": 0.5217, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.4584256410598755, + "rewards/margins": 0.8441031575202942, + "rewards/rejected": -2.3025288581848145, + "step": 5190 + }, + { + "epoch": 0.34, + "learning_rate": 4.171553136223222e-06, + "logits/chosen": -1.3787262439727783, + "logits/rejected": -1.2474148273468018, + "logps/chosen": -437.20831298828125, + "logps/rejected": -550.6790771484375, + "loss": 0.4666, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6988741159439087, + "rewards/margins": 1.0276092290878296, + "rewards/rejected": -2.7264835834503174, + "step": 5200 + }, + { + "epoch": 0.34, + "eval_logits/chosen": -1.1704281568527222, + "eval_logits/rejected": -1.0363733768463135, + "eval_logps/chosen": -391.68426513671875, + "eval_logps/rejected": -446.65252685546875, + "eval_loss": 0.5153765678405762, + "eval_rewards/accuracies": 0.6600000262260437, + "eval_rewards/chosen": -1.5967929363250732, + "eval_rewards/margins": 0.7536134123802185, + "eval_rewards/rejected": -2.3504061698913574, + "eval_runtime": 733.295, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.364, + "step": 5200 + }, + { + "epoch": 0.34, + "learning_rate": 4.167302872902865e-06, + "logits/chosen": -1.2382739782333374, + "logits/rejected": -0.9230550527572632, + "logps/chosen": -414.4139709472656, + "logps/rejected": -526.5465698242188, + "loss": 0.4884, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6294580698013306, + "rewards/margins": 1.2407068014144897, + "rewards/rejected": -2.8701653480529785, + "step": 5210 + }, + { + "epoch": 0.34, + "learning_rate": 4.163043912113985e-06, + "logits/chosen": -1.1802736520767212, + "logits/rejected": -0.9871931076049805, + "logps/chosen": -414.599609375, + "logps/rejected": -441.56396484375, + "loss": 0.5555, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.555594801902771, + "rewards/margins": 0.6095894575119019, + "rewards/rejected": -2.165184259414673, + "step": 5220 + }, + { + "epoch": 0.34, + "learning_rate": 4.15877627607341e-06, + "logits/chosen": -0.9541276097297668, + "logits/rejected": -0.7061694860458374, + "logps/chosen": -375.0079345703125, + "logps/rejected": -419.3385314941406, + "loss": 0.5109, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.5841313600540161, + "rewards/margins": 0.6855292916297913, + "rewards/rejected": -2.269660711288452, + "step": 5230 + }, + { + "epoch": 0.34, + "learning_rate": 4.154499987043217e-06, + "logits/chosen": -1.1777527332305908, + "logits/rejected": -0.9939281344413757, + "logps/chosen": -361.29815673828125, + "logps/rejected": -444.8980407714844, + "loss": 0.4545, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.3709460496902466, + "rewards/margins": 1.0577964782714844, + "rewards/rejected": -2.4287424087524414, + "step": 5240 + }, + { + "epoch": 0.34, + "learning_rate": 4.150215067330625e-06, + "logits/chosen": -1.099338173866272, + "logits/rejected": -0.8791137933731079, + "logps/chosen": -346.3368225097656, + "logps/rejected": -462.539794921875, + "loss": 0.4998, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.372879981994629, + "rewards/margins": 1.0216567516326904, + "rewards/rejected": -2.3945367336273193, + "step": 5250 + }, + { + "epoch": 0.34, + "learning_rate": 4.145921539287876e-06, + "logits/chosen": -1.0073109865188599, + "logits/rejected": -0.9385402798652649, + "logps/chosen": -321.3467102050781, + "logps/rejected": -424.44293212890625, + "loss": 0.4308, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.3075411319732666, + "rewards/margins": 1.2309949398040771, + "rewards/rejected": -2.5385358333587646, + "step": 5260 + }, + { + "epoch": 0.34, + "learning_rate": 4.141619425312115e-06, + "logits/chosen": -1.014169454574585, + "logits/rejected": -0.7141678333282471, + "logps/chosen": -381.5502624511719, + "logps/rejected": -399.2060241699219, + "loss": 0.6727, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -1.7289901971817017, + "rewards/margins": 0.420976459980011, + "rewards/rejected": -2.1499664783477783, + "step": 5270 + }, + { + "epoch": 0.35, + "learning_rate": 4.1373087478452735e-06, + "logits/chosen": -0.9243456125259399, + "logits/rejected": -0.8291029930114746, + "logps/chosen": -372.74444580078125, + "logps/rejected": -466.50909423828125, + "loss": 0.3357, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.4750388860702515, + "rewards/margins": 1.4837852716445923, + "rewards/rejected": -2.9588239192962646, + "step": 5280 + }, + { + "epoch": 0.35, + "learning_rate": 4.132989529373959e-06, + "logits/chosen": -0.9148101806640625, + "logits/rejected": -0.732371985912323, + "logps/chosen": -496.69793701171875, + "logps/rejected": -481.1084899902344, + "loss": 0.5406, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.407851219177246, + "rewards/margins": 0.634090781211853, + "rewards/rejected": -3.0419418811798096, + "step": 5290 + }, + { + "epoch": 0.35, + "learning_rate": 4.128661792429331e-06, + "logits/chosen": -0.8414371609687805, + "logits/rejected": -0.7446430325508118, + "logps/chosen": -479.20733642578125, + "logps/rejected": -548.0040893554688, + "loss": 0.6107, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.2543675899505615, + "rewards/margins": 0.6390036344528198, + "rewards/rejected": -2.893371105194092, + "step": 5300 + }, + { + "epoch": 0.35, + "eval_logits/chosen": -0.9356719851493835, + "eval_logits/rejected": -0.8068389892578125, + "eval_logps/chosen": -456.3209228515625, + "eval_logps/rejected": -521.69482421875, + "eval_loss": 0.5146012902259827, + "eval_rewards/accuracies": 0.6570000052452087, + "eval_rewards/chosen": -2.2431600093841553, + "eval_rewards/margins": 0.8576699495315552, + "eval_rewards/rejected": -3.100829601287842, + "eval_runtime": 732.9797, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.364, + "step": 5300 + }, + { + "epoch": 0.35, + "learning_rate": 4.124325559586985e-06, + "logits/chosen": -0.9261112213134766, + "logits/rejected": -0.7737448811531067, + "logps/chosen": -415.0210876464844, + "logps/rejected": -464.01275634765625, + "loss": 0.8043, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.269643545150757, + "rewards/margins": 0.3606763482093811, + "rewards/rejected": -2.630319833755493, + "step": 5310 + }, + { + "epoch": 0.35, + "learning_rate": 4.119980853466835e-06, + "logits/chosen": -0.700569212436676, + "logits/rejected": -0.3479693830013275, + "logps/chosen": -467.34918212890625, + "logps/rejected": -507.01568603515625, + "loss": 0.4996, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.5819480419158936, + "rewards/margins": 0.6678058505058289, + "rewards/rejected": -3.249753952026367, + "step": 5320 + }, + { + "epoch": 0.35, + "learning_rate": 4.115627696732997e-06, + "logits/chosen": -0.7397909760475159, + "logits/rejected": -0.6655017733573914, + "logps/chosen": -402.98309326171875, + "logps/rejected": -443.57208251953125, + "loss": 0.5533, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.1156888008117676, + "rewards/margins": 0.5990991592407227, + "rewards/rejected": -2.714787721633911, + "step": 5330 + }, + { + "epoch": 0.35, + "learning_rate": 4.111266112093668e-06, + "logits/chosen": -0.9121619462966919, + "logits/rejected": -0.7928305268287659, + "logps/chosen": -454.974609375, + "logps/rejected": -538.3798217773438, + "loss": 0.5456, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.5150694847106934, + "rewards/margins": 0.6500739455223083, + "rewards/rejected": -3.1651434898376465, + "step": 5340 + }, + { + "epoch": 0.35, + "learning_rate": 4.1068961223010115e-06, + "logits/chosen": -1.0712369680404663, + "logits/rejected": -0.616116464138031, + "logps/chosen": -550.204345703125, + "logps/rejected": -586.81591796875, + "loss": 0.6113, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.7950360774993896, + "rewards/margins": 0.5928794145584106, + "rewards/rejected": -3.387915849685669, + "step": 5350 + }, + { + "epoch": 0.35, + "learning_rate": 4.102517750151034e-06, + "logits/chosen": -1.1240382194519043, + "logits/rejected": -0.8812620043754578, + "logps/chosen": -509.80694580078125, + "logps/rejected": -496.1722106933594, + "loss": 0.5562, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.1476213932037354, + "rewards/margins": 0.5767813920974731, + "rewards/rejected": -2.724402666091919, + "step": 5360 + }, + { + "epoch": 0.35, + "learning_rate": 4.09813101848347e-06, + "logits/chosen": -1.180262804031372, + "logits/rejected": -0.9633585810661316, + "logps/chosen": -409.8777770996094, + "logps/rejected": -507.8788146972656, + "loss": 0.5133, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9089186191558838, + "rewards/margins": 0.8084686398506165, + "rewards/rejected": -2.7173869609832764, + "step": 5370 + }, + { + "epoch": 0.35, + "learning_rate": 4.093735950181659e-06, + "logits/chosen": -1.004286527633667, + "logits/rejected": -0.8687463998794556, + "logps/chosen": -397.5352783203125, + "logps/rejected": -523.0494384765625, + "loss": 0.4518, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7206192016601562, + "rewards/margins": 1.0097081661224365, + "rewards/rejected": -2.7303271293640137, + "step": 5380 + }, + { + "epoch": 0.35, + "learning_rate": 4.0893325681724326e-06, + "logits/chosen": -1.2412950992584229, + "logits/rejected": -1.1470402479171753, + "logps/chosen": -440.41473388671875, + "logps/rejected": -540.3668212890625, + "loss": 0.4247, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.8199307918548584, + "rewards/margins": 1.1223140954971313, + "rewards/rejected": -2.9422447681427, + "step": 5390 + }, + { + "epoch": 0.35, + "learning_rate": 4.084920895425988e-06, + "logits/chosen": -1.1176074743270874, + "logits/rejected": -0.9556187391281128, + "logps/chosen": -450.7571716308594, + "logps/rejected": -545.2280883789062, + "loss": 0.5853, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.133924722671509, + "rewards/margins": 0.8082602620124817, + "rewards/rejected": -2.9421849250793457, + "step": 5400 + }, + { + "epoch": 0.35, + "eval_logits/chosen": -1.0984172821044922, + "eval_logits/rejected": -0.9615691900253296, + "eval_logps/chosen": -401.5628662109375, + "eval_logps/rejected": -471.24493408203125, + "eval_loss": 0.5090234875679016, + "eval_rewards/accuracies": 0.6625000238418579, + "eval_rewards/chosen": -1.695579171180725, + "eval_rewards/margins": 0.9007511734962463, + "eval_rewards/rejected": -2.596330404281616, + "eval_runtime": 733.7513, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 5400 + }, + { + "epoch": 0.35, + "learning_rate": 4.080500954955769e-06, + "logits/chosen": -1.0121065378189087, + "logits/rejected": -0.8766088485717773, + "logps/chosen": -451.9566955566406, + "logps/rejected": -547.6082153320312, + "loss": 0.5004, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9608478546142578, + "rewards/margins": 1.1393764019012451, + "rewards/rejected": -3.100224256515503, + "step": 5410 + }, + { + "epoch": 0.35, + "learning_rate": 4.076072769818354e-06, + "logits/chosen": -1.373978853225708, + "logits/rejected": -1.0941734313964844, + "logps/chosen": -385.83087158203125, + "logps/rejected": -430.9171447753906, + "loss": 0.4115, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.421086072921753, + "rewards/margins": 1.0705190896987915, + "rewards/rejected": -2.491605043411255, + "step": 5420 + }, + { + "epoch": 0.36, + "learning_rate": 4.071636363113323e-06, + "logits/chosen": -0.8405832052230835, + "logits/rejected": -0.7371835708618164, + "logps/chosen": -422.33856201171875, + "logps/rejected": -424.07147216796875, + "loss": 0.659, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.5420846939086914, + "rewards/margins": 0.5577548742294312, + "rewards/rejected": -2.099839687347412, + "step": 5430 + }, + { + "epoch": 0.36, + "learning_rate": 4.067191757983146e-06, + "logits/chosen": -0.9356991648674011, + "logits/rejected": -0.8254088163375854, + "logps/chosen": -376.23724365234375, + "logps/rejected": -477.99822998046875, + "loss": 0.4982, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.466615915298462, + "rewards/margins": 1.0497499704360962, + "rewards/rejected": -2.5163657665252686, + "step": 5440 + }, + { + "epoch": 0.36, + "learning_rate": 4.062738977613063e-06, + "logits/chosen": -0.7855893969535828, + "logits/rejected": -0.8330990672111511, + "logps/chosen": -376.4490966796875, + "logps/rejected": -393.9264831542969, + "loss": 0.5883, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.4197274446487427, + "rewards/margins": 0.6587068438529968, + "rewards/rejected": -2.078434467315674, + "step": 5450 + }, + { + "epoch": 0.36, + "learning_rate": 4.058278045230957e-06, + "logits/chosen": -1.189239501953125, + "logits/rejected": -1.1592838764190674, + "logps/chosen": -382.54998779296875, + "logps/rejected": -449.02276611328125, + "loss": 0.601, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.641831398010254, + "rewards/margins": 0.6488510370254517, + "rewards/rejected": -2.290682315826416, + "step": 5460 + }, + { + "epoch": 0.36, + "learning_rate": 4.053808984107235e-06, + "logits/chosen": -1.1778719425201416, + "logits/rejected": -1.030869960784912, + "logps/chosen": -386.10565185546875, + "logps/rejected": -389.18975830078125, + "loss": 0.6179, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.5323729515075684, + "rewards/margins": 0.43377256393432617, + "rewards/rejected": -1.9661457538604736, + "step": 5470 + }, + { + "epoch": 0.36, + "learning_rate": 4.04933181755471e-06, + "logits/chosen": -1.2040650844573975, + "logits/rejected": -1.190525770187378, + "logps/chosen": -352.6644287109375, + "logps/rejected": -414.3978576660156, + "loss": 0.5128, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.460837960243225, + "rewards/margins": 0.6894599199295044, + "rewards/rejected": -2.1502978801727295, + "step": 5480 + }, + { + "epoch": 0.36, + "learning_rate": 4.044846568928477e-06, + "logits/chosen": -1.382424235343933, + "logits/rejected": -1.3112528324127197, + "logps/chosen": -436.9786071777344, + "logps/rejected": -498.826171875, + "loss": 0.5284, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7545440196990967, + "rewards/margins": 0.7070547938346863, + "rewards/rejected": -2.461599111557007, + "step": 5490 + }, + { + "epoch": 0.36, + "learning_rate": 4.040353261625788e-06, + "logits/chosen": -1.389473557472229, + "logits/rejected": -0.8910207748413086, + "logps/chosen": -446.0039978027344, + "logps/rejected": -486.54071044921875, + "loss": 0.5086, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.6945879459381104, + "rewards/margins": 0.7916721701622009, + "rewards/rejected": -2.486259937286377, + "step": 5500 + }, + { + "epoch": 0.36, + "eval_logits/chosen": -1.1007320880889893, + "eval_logits/rejected": -0.9732699990272522, + "eval_logps/chosen": -405.7402648925781, + "eval_logps/rejected": -457.79937744140625, + "eval_loss": 0.5213733911514282, + "eval_rewards/accuracies": 0.659500002861023, + "eval_rewards/chosen": -1.7373523712158203, + "eval_rewards/margins": 0.7245224118232727, + "eval_rewards/rejected": -2.4618749618530273, + "eval_runtime": 733.2339, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 5500 + }, + { + "epoch": 0.36, + "learning_rate": 4.035851919085936e-06, + "logits/chosen": -1.179037094116211, + "logits/rejected": -0.9055557250976562, + "logps/chosen": -443.47998046875, + "logps/rejected": -447.66864013671875, + "loss": 0.5885, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7608587741851807, + "rewards/margins": 0.8290047645568848, + "rewards/rejected": -2.5898633003234863, + "step": 5510 + }, + { + "epoch": 0.36, + "learning_rate": 4.031342564790128e-06, + "logits/chosen": -1.1599061489105225, + "logits/rejected": -0.9323163032531738, + "logps/chosen": -360.9304504394531, + "logps/rejected": -445.9639587402344, + "loss": 0.4878, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.5495872497558594, + "rewards/margins": 0.8980630040168762, + "rewards/rejected": -2.44765043258667, + "step": 5520 + }, + { + "epoch": 0.36, + "learning_rate": 4.026825222261367e-06, + "logits/chosen": -0.9034293293952942, + "logits/rejected": -0.714095413684845, + "logps/chosen": -382.3951110839844, + "logps/rejected": -428.35223388671875, + "loss": 0.6408, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.093200922012329, + "rewards/margins": 0.5310032963752747, + "rewards/rejected": -2.624203681945801, + "step": 5530 + }, + { + "epoch": 0.36, + "learning_rate": 4.022299915064321e-06, + "logits/chosen": -1.158623218536377, + "logits/rejected": -0.9449752569198608, + "logps/chosen": -453.1712341308594, + "logps/rejected": -479.60687255859375, + "loss": 0.4749, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.4732328653335571, + "rewards/margins": 0.795411229133606, + "rewards/rejected": -2.268643856048584, + "step": 5540 + }, + { + "epoch": 0.36, + "learning_rate": 4.017766666805213e-06, + "logits/chosen": -1.106878638267517, + "logits/rejected": -0.8825507164001465, + "logps/chosen": -366.5860595703125, + "logps/rejected": -397.0874328613281, + "loss": 0.6049, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.5306566953659058, + "rewards/margins": 0.6586295366287231, + "rewards/rejected": -2.189286470413208, + "step": 5550 + }, + { + "epoch": 0.36, + "learning_rate": 4.013225501131684e-06, + "logits/chosen": -1.176776647567749, + "logits/rejected": -0.9218536615371704, + "logps/chosen": -389.47491455078125, + "logps/rejected": -412.9156799316406, + "loss": 0.5809, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.720973253250122, + "rewards/margins": 0.5041357278823853, + "rewards/rejected": -2.225109100341797, + "step": 5560 + }, + { + "epoch": 0.36, + "learning_rate": 4.008676441732679e-06, + "logits/chosen": -0.9157624244689941, + "logits/rejected": -0.7230433821678162, + "logps/chosen": -374.0820007324219, + "logps/rejected": -383.73028564453125, + "loss": 0.5511, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.6778764724731445, + "rewards/margins": 0.5764918327331543, + "rewards/rejected": -2.254368305206299, + "step": 5570 + }, + { + "epoch": 0.37, + "learning_rate": 4.00411951233832e-06, + "logits/chosen": -1.112180471420288, + "logits/rejected": -0.9340667724609375, + "logps/chosen": -386.84442138671875, + "logps/rejected": -427.77093505859375, + "loss": 0.4714, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.6711927652359009, + "rewards/margins": 0.8560009002685547, + "rewards/rejected": -2.527193307876587, + "step": 5580 + }, + { + "epoch": 0.37, + "learning_rate": 3.999554736719785e-06, + "logits/chosen": -0.8898111581802368, + "logits/rejected": -0.8584083318710327, + "logps/chosen": -453.03466796875, + "logps/rejected": -473.88739013671875, + "loss": 0.5457, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6105778217315674, + "rewards/margins": 0.6437070369720459, + "rewards/rejected": -2.2542850971221924, + "step": 5590 + }, + { + "epoch": 0.37, + "learning_rate": 3.994982138689177e-06, + "logits/chosen": -1.4541324377059937, + "logits/rejected": -1.138900637626648, + "logps/chosen": -391.6158752441406, + "logps/rejected": -467.52142333984375, + "loss": 0.4764, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.5421569347381592, + "rewards/margins": 0.7775677442550659, + "rewards/rejected": -2.3197245597839355, + "step": 5600 + }, + { + "epoch": 0.37, + "eval_logits/chosen": -1.0609233379364014, + "eval_logits/rejected": -0.9316677451133728, + "eval_logps/chosen": -393.97259521484375, + "eval_logps/rejected": -452.8468322753906, + "eval_loss": 0.5123740434646606, + "eval_rewards/accuracies": 0.6625000238418579, + "eval_rewards/chosen": -1.6196763515472412, + "eval_rewards/margins": 0.792672872543335, + "eval_rewards/rejected": -2.412349224090576, + "eval_runtime": 732.7002, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 5600 + }, + { + "epoch": 0.37, + "learning_rate": 3.990401742099408e-06, + "logits/chosen": -0.8740717768669128, + "logits/rejected": -0.8899892568588257, + "logps/chosen": -323.3037414550781, + "logps/rejected": -371.63824462890625, + "loss": 0.6185, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -1.4444063901901245, + "rewards/margins": 0.556832492351532, + "rewards/rejected": -2.0012388229370117, + "step": 5610 + }, + { + "epoch": 0.37, + "learning_rate": 3.985813570844072e-06, + "logits/chosen": -1.2019202709197998, + "logits/rejected": -1.0983178615570068, + "logps/chosen": -469.497314453125, + "logps/rejected": -519.1656494140625, + "loss": 0.6347, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7511056661605835, + "rewards/margins": 0.7721987962722778, + "rewards/rejected": -2.5233044624328613, + "step": 5620 + }, + { + "epoch": 0.37, + "learning_rate": 3.981217648857316e-06, + "logits/chosen": -1.1407134532928467, + "logits/rejected": -1.063767433166504, + "logps/chosen": -300.8212890625, + "logps/rejected": -393.89117431640625, + "loss": 0.4415, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.2828083038330078, + "rewards/margins": 0.9128808975219727, + "rewards/rejected": -2.1956894397735596, + "step": 5630 + }, + { + "epoch": 0.37, + "learning_rate": 3.97661400011372e-06, + "logits/chosen": -1.126928448677063, + "logits/rejected": -1.2266345024108887, + "logps/chosen": -359.68841552734375, + "logps/rejected": -416.9710388183594, + "loss": 0.5515, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.1666107177734375, + "rewards/margins": 0.6870548725128174, + "rewards/rejected": -1.8536655902862549, + "step": 5640 + }, + { + "epoch": 0.37, + "learning_rate": 3.972002648628174e-06, + "logits/chosen": -1.192249059677124, + "logits/rejected": -1.0249332189559937, + "logps/chosen": -435.70001220703125, + "logps/rejected": -451.20928955078125, + "loss": 0.5513, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.5866916179656982, + "rewards/margins": 0.6320112943649292, + "rewards/rejected": -2.218702793121338, + "step": 5650 + }, + { + "epoch": 0.37, + "learning_rate": 3.967383618455743e-06, + "logits/chosen": -1.167616605758667, + "logits/rejected": -1.2211428880691528, + "logps/chosen": -407.02044677734375, + "logps/rejected": -497.1388244628906, + "loss": 0.5616, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7990070581436157, + "rewards/margins": 0.719279408454895, + "rewards/rejected": -2.5182864665985107, + "step": 5660 + }, + { + "epoch": 0.37, + "learning_rate": 3.9627569336915515e-06, + "logits/chosen": -1.4768421649932861, + "logits/rejected": -1.2465412616729736, + "logps/chosen": -409.7810974121094, + "logps/rejected": -429.510009765625, + "loss": 0.5254, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.623183250427246, + "rewards/margins": 0.7640554308891296, + "rewards/rejected": -2.3872387409210205, + "step": 5670 + }, + { + "epoch": 0.37, + "learning_rate": 3.9581226184706555e-06, + "logits/chosen": -1.3483765125274658, + "logits/rejected": -1.5304193496704102, + "logps/chosen": -384.47637939453125, + "logps/rejected": -524.4840698242188, + "loss": 0.5158, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8981096744537354, + "rewards/margins": 0.6931003332138062, + "rewards/rejected": -2.591209888458252, + "step": 5680 + }, + { + "epoch": 0.37, + "learning_rate": 3.953480696967912e-06, + "logits/chosen": -1.0088798999786377, + "logits/rejected": -1.1363639831542969, + "logps/chosen": -370.19451904296875, + "logps/rejected": -457.1572265625, + "loss": 0.5491, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.608229398727417, + "rewards/margins": 0.5051697492599487, + "rewards/rejected": -2.113399028778076, + "step": 5690 + }, + { + "epoch": 0.37, + "learning_rate": 3.948831193397857e-06, + "logits/chosen": -1.0741056203842163, + "logits/rejected": -1.0835530757904053, + "logps/chosen": -304.1203308105469, + "logps/rejected": -343.5845642089844, + "loss": 0.6562, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3810492753982544, + "rewards/margins": 0.3363833725452423, + "rewards/rejected": -1.7174327373504639, + "step": 5700 + }, + { + "epoch": 0.37, + "eval_logits/chosen": -1.2059582471847534, + "eval_logits/rejected": -1.0711474418640137, + "eval_logps/chosen": -369.1748962402344, + "eval_logps/rejected": -425.8072509765625, + "eval_loss": 0.5096937417984009, + "eval_rewards/accuracies": 0.6710000038146973, + "eval_rewards/chosen": -1.371699333190918, + "eval_rewards/margins": 0.7702544331550598, + "eval_rewards/rejected": -2.141953706741333, + "eval_runtime": 733.1703, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 5700 + }, + { + "epoch": 0.37, + "learning_rate": 3.94417413201458e-06, + "logits/chosen": -1.284786343574524, + "logits/rejected": -1.0199940204620361, + "logps/chosen": -344.146240234375, + "logps/rejected": -395.18060302734375, + "loss": 0.5787, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.3257472515106201, + "rewards/margins": 0.6880120038986206, + "rewards/rejected": -2.013759136199951, + "step": 5710 + }, + { + "epoch": 0.37, + "learning_rate": 3.9395095371115935e-06, + "logits/chosen": -1.1523475646972656, + "logits/rejected": -1.065656065940857, + "logps/chosen": -355.04437255859375, + "logps/rejected": -424.20001220703125, + "loss": 0.5606, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.423872947692871, + "rewards/margins": 0.7976680994033813, + "rewards/rejected": -2.221540927886963, + "step": 5720 + }, + { + "epoch": 0.37, + "learning_rate": 3.93483743302171e-06, + "logits/chosen": -1.2104334831237793, + "logits/rejected": -0.918979823589325, + "logps/chosen": -350.27996826171875, + "logps/rejected": -402.14666748046875, + "loss": 0.5375, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3748830556869507, + "rewards/margins": 0.7560206055641174, + "rewards/rejected": -2.130903720855713, + "step": 5730 + }, + { + "epoch": 0.38, + "learning_rate": 3.930157844116913e-06, + "logits/chosen": -1.1220810413360596, + "logits/rejected": -0.8528586626052856, + "logps/chosen": -365.50885009765625, + "logps/rejected": -416.3897399902344, + "loss": 0.6294, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.5575157403945923, + "rewards/margins": 0.6559960246086121, + "rewards/rejected": -2.2135117053985596, + "step": 5740 + }, + { + "epoch": 0.38, + "learning_rate": 3.925470794808229e-06, + "logits/chosen": -1.0437357425689697, + "logits/rejected": -0.9060714840888977, + "logps/chosen": -393.3031005859375, + "logps/rejected": -465.1321716308594, + "loss": 0.4304, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.5431537628173828, + "rewards/margins": 1.046464443206787, + "rewards/rejected": -2.58961820602417, + "step": 5750 + }, + { + "epoch": 0.38, + "learning_rate": 3.920776309545606e-06, + "logits/chosen": -1.3418052196502686, + "logits/rejected": -1.122631311416626, + "logps/chosen": -249.40090942382812, + "logps/rejected": -329.80426025390625, + "loss": 0.4853, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.9669389724731445, + "rewards/margins": 0.8723586797714233, + "rewards/rejected": -1.8392976522445679, + "step": 5760 + }, + { + "epoch": 0.38, + "learning_rate": 3.916074412817778e-06, + "logits/chosen": -1.1772065162658691, + "logits/rejected": -0.8717998266220093, + "logps/chosen": -393.0423278808594, + "logps/rejected": -518.3782958984375, + "loss": 0.3867, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.497083067893982, + "rewards/margins": 1.2481367588043213, + "rewards/rejected": -2.7452197074890137, + "step": 5770 + }, + { + "epoch": 0.38, + "learning_rate": 3.911365129152139e-06, + "logits/chosen": -1.2302935123443604, + "logits/rejected": -1.1185319423675537, + "logps/chosen": -387.8710632324219, + "logps/rejected": -487.1651916503906, + "loss": 0.507, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5543718338012695, + "rewards/margins": 1.0483394861221313, + "rewards/rejected": -2.6027112007141113, + "step": 5780 + }, + { + "epoch": 0.38, + "learning_rate": 3.906648483114623e-06, + "logits/chosen": -1.241497278213501, + "logits/rejected": -1.0616934299468994, + "logps/chosen": -333.7834777832031, + "logps/rejected": -418.350830078125, + "loss": 0.5069, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.364598274230957, + "rewards/margins": 1.171523928642273, + "rewards/rejected": -2.5361220836639404, + "step": 5790 + }, + { + "epoch": 0.38, + "learning_rate": 3.901924499309564e-06, + "logits/chosen": -1.1808868646621704, + "logits/rejected": -1.052524209022522, + "logps/chosen": -348.7895202636719, + "logps/rejected": -419.6104431152344, + "loss": 0.5178, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2008459568023682, + "rewards/margins": 1.0342692136764526, + "rewards/rejected": -2.2351150512695312, + "step": 5800 + }, + { + "epoch": 0.38, + "eval_logits/chosen": -1.2822357416152954, + "eval_logits/rejected": -1.1354150772094727, + "eval_logps/chosen": -367.5433044433594, + "eval_logps/rejected": -447.6251220703125, + "eval_loss": 0.5039393305778503, + "eval_rewards/accuracies": 0.6614999771118164, + "eval_rewards/chosen": -1.3553833961486816, + "eval_rewards/margins": 1.0047485828399658, + "eval_rewards/rejected": -2.3601317405700684, + "eval_runtime": 733.8198, + "eval_samples_per_second": 2.725, + "eval_steps_per_second": 1.363, + "step": 5800 + }, + { + "epoch": 0.38, + "learning_rate": 3.897193202379575e-06, + "logits/chosen": -1.3546619415283203, + "logits/rejected": -1.1554213762283325, + "logps/chosen": -323.00579833984375, + "logps/rejected": -405.39202880859375, + "loss": 0.547, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.225281000137329, + "rewards/margins": 0.9935903549194336, + "rewards/rejected": -2.2188713550567627, + "step": 5810 + }, + { + "epoch": 0.38, + "learning_rate": 3.8924546170054215e-06, + "logits/chosen": -1.2916877269744873, + "logits/rejected": -1.132554054260254, + "logps/chosen": -355.0088806152344, + "logps/rejected": -439.87890625, + "loss": 0.5571, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3910125494003296, + "rewards/margins": 1.0434343814849854, + "rewards/rejected": -2.4344468116760254, + "step": 5820 + }, + { + "epoch": 0.38, + "learning_rate": 3.887708767905883e-06, + "logits/chosen": -1.4169222116470337, + "logits/rejected": -1.1427775621414185, + "logps/chosen": -397.4792175292969, + "logps/rejected": -421.56280517578125, + "loss": 0.4589, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5371202230453491, + "rewards/margins": 0.8992649912834167, + "rewards/rejected": -2.4363853931427, + "step": 5830 + }, + { + "epoch": 0.38, + "learning_rate": 3.882955679837636e-06, + "logits/chosen": -1.1456177234649658, + "logits/rejected": -1.1761143207550049, + "logps/chosen": -452.54736328125, + "logps/rejected": -531.5821533203125, + "loss": 0.5629, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.1317031383514404, + "rewards/margins": 0.7665945887565613, + "rewards/rejected": -2.8982975482940674, + "step": 5840 + }, + { + "epoch": 0.38, + "learning_rate": 3.878195377595113e-06, + "logits/chosen": -1.1690410375595093, + "logits/rejected": -1.02626371383667, + "logps/chosen": -395.9454650878906, + "logps/rejected": -514.1718139648438, + "loss": 0.5532, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.600396752357483, + "rewards/margins": 1.2378990650177002, + "rewards/rejected": -2.8382959365844727, + "step": 5850 + }, + { + "epoch": 0.38, + "learning_rate": 3.873427886010384e-06, + "logits/chosen": -1.234686255455017, + "logits/rejected": -0.9635084867477417, + "logps/chosen": -356.0743408203125, + "logps/rejected": -423.4266052246094, + "loss": 0.5384, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.656224012374878, + "rewards/margins": 0.8644134402275085, + "rewards/rejected": -2.5206375122070312, + "step": 5860 + }, + { + "epoch": 0.38, + "learning_rate": 3.868653229953021e-06, + "logits/chosen": -1.2322075366973877, + "logits/rejected": -1.0185177326202393, + "logps/chosen": -443.06982421875, + "logps/rejected": -576.76123046875, + "loss": 0.4052, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0585498809814453, + "rewards/margins": 1.357552170753479, + "rewards/rejected": -3.416102170944214, + "step": 5870 + }, + { + "epoch": 0.38, + "learning_rate": 3.8638714343299675e-06, + "logits/chosen": -1.262803316116333, + "logits/rejected": -1.1337450742721558, + "logps/chosen": -364.6521911621094, + "logps/rejected": -484.88433837890625, + "loss": 0.4626, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.463094711303711, + "rewards/margins": 1.0511635541915894, + "rewards/rejected": -2.5142581462860107, + "step": 5880 + }, + { + "epoch": 0.39, + "learning_rate": 3.859082524085414e-06, + "logits/chosen": -1.118187665939331, + "logits/rejected": -0.9261728525161743, + "logps/chosen": -433.3507385253906, + "logps/rejected": -452.5609436035156, + "loss": 0.5653, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.61007559299469, + "rewards/margins": 0.7944930791854858, + "rewards/rejected": -2.404569149017334, + "step": 5890 + }, + { + "epoch": 0.39, + "learning_rate": 3.854286524200659e-06, + "logits/chosen": -1.5227950811386108, + "logits/rejected": -1.1201550960540771, + "logps/chosen": -422.0400390625, + "logps/rejected": -453.4849548339844, + "loss": 0.5391, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.4562262296676636, + "rewards/margins": 0.8208130598068237, + "rewards/rejected": -2.2770392894744873, + "step": 5900 + }, + { + "epoch": 0.39, + "eval_logits/chosen": -1.2484019994735718, + "eval_logits/rejected": -1.1068450212478638, + "eval_logps/chosen": -369.74603271484375, + "eval_logps/rejected": -439.0063171386719, + "eval_loss": 0.503896176815033, + "eval_rewards/accuracies": 0.6614999771118164, + "eval_rewards/chosen": -1.3774104118347168, + "eval_rewards/margins": 0.8965333700180054, + "eval_rewards/rejected": -2.273944139480591, + "eval_runtime": 733.8267, + "eval_samples_per_second": 2.725, + "eval_steps_per_second": 1.363, + "step": 5900 + }, + { + "epoch": 0.39, + "learning_rate": 3.849483459693991e-06, + "logits/chosen": -1.343681812286377, + "logits/rejected": -1.1354031562805176, + "logps/chosen": -343.40374755859375, + "logps/rejected": -400.17694091796875, + "loss": 0.4648, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.3372704982757568, + "rewards/margins": 0.9544233083724976, + "rewards/rejected": -2.291693687438965, + "step": 5910 + }, + { + "epoch": 0.39, + "learning_rate": 3.844673355620544e-06, + "logits/chosen": -1.2277530431747437, + "logits/rejected": -0.9738208651542664, + "logps/chosen": -422.9916076660156, + "logps/rejected": -475.74822998046875, + "loss": 0.5377, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.7641870975494385, + "rewards/margins": 0.9104213714599609, + "rewards/rejected": -2.6746084690093994, + "step": 5920 + }, + { + "epoch": 0.39, + "learning_rate": 3.839856237072178e-06, + "logits/chosen": -0.9866417646408081, + "logits/rejected": -0.9235474467277527, + "logps/chosen": -350.86114501953125, + "logps/rejected": -475.2818298339844, + "loss": 0.5043, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.6828702688217163, + "rewards/margins": 1.072285771369934, + "rewards/rejected": -2.7551560401916504, + "step": 5930 + }, + { + "epoch": 0.39, + "learning_rate": 3.8350321291773455e-06, + "logits/chosen": -0.9643562436103821, + "logits/rejected": -0.8080571293830872, + "logps/chosen": -331.8020324707031, + "logps/rejected": -392.1284484863281, + "loss": 0.5131, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.2928006649017334, + "rewards/margins": 1.04522705078125, + "rewards/rejected": -2.3380279541015625, + "step": 5940 + }, + { + "epoch": 0.39, + "learning_rate": 3.830201057100953e-06, + "logits/chosen": -1.3062032461166382, + "logits/rejected": -1.205256462097168, + "logps/chosen": -359.8435974121094, + "logps/rejected": -475.44189453125, + "loss": 0.4919, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6807063817977905, + "rewards/margins": 1.0010545253753662, + "rewards/rejected": -2.681760787963867, + "step": 5950 + }, + { + "epoch": 0.39, + "learning_rate": 3.82536304604424e-06, + "logits/chosen": -1.07782781124115, + "logits/rejected": -0.9527738690376282, + "logps/chosen": -372.5465087890625, + "logps/rejected": -422.93499755859375, + "loss": 0.531, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3638226985931396, + "rewards/margins": 0.8212461471557617, + "rewards/rejected": -2.1850686073303223, + "step": 5960 + }, + { + "epoch": 0.39, + "learning_rate": 3.8205181212446435e-06, + "logits/chosen": -1.4037820100784302, + "logits/rejected": -1.2214797735214233, + "logps/chosen": -376.4328918457031, + "logps/rejected": -438.6416931152344, + "loss": 0.4591, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.0587395429611206, + "rewards/margins": 1.0261236429214478, + "rewards/rejected": -2.0848631858825684, + "step": 5970 + }, + { + "epoch": 0.39, + "learning_rate": 3.815666307975664e-06, + "logits/chosen": -1.19920015335083, + "logits/rejected": -1.1759016513824463, + "logps/chosen": -379.8518981933594, + "logps/rejected": -470.53643798828125, + "loss": 0.4468, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.425341010093689, + "rewards/margins": 1.09866464138031, + "rewards/rejected": -2.524005651473999, + "step": 5980 + }, + { + "epoch": 0.39, + "learning_rate": 3.8108076315467346e-06, + "logits/chosen": -1.3615524768829346, + "logits/rejected": -1.2437620162963867, + "logps/chosen": -436.19671630859375, + "logps/rejected": -463.3844299316406, + "loss": 0.548, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.789139747619629, + "rewards/margins": 1.0191426277160645, + "rewards/rejected": -2.8082823753356934, + "step": 5990 + }, + { + "epoch": 0.39, + "learning_rate": 3.805942117303093e-06, + "logits/chosen": -1.3936203718185425, + "logits/rejected": -1.2214877605438232, + "logps/chosen": -477.30487060546875, + "logps/rejected": -533.3314819335938, + "loss": 0.4757, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.612693190574646, + "rewards/margins": 1.0255014896392822, + "rewards/rejected": -2.6381945610046387, + "step": 6000 + }, + { + "epoch": 0.39, + "eval_logits/chosen": -1.0945810079574585, + "eval_logits/rejected": -0.9610912799835205, + "eval_logps/chosen": -386.2829284667969, + "eval_logps/rejected": -458.74658203125, + "eval_loss": 0.5028457045555115, + "eval_rewards/accuracies": 0.6654999852180481, + "eval_rewards/chosen": -1.5427796840667725, + "eval_rewards/margins": 0.9285673499107361, + "eval_rewards/rejected": -2.4713470935821533, + "eval_runtime": 734.4828, + "eval_samples_per_second": 2.723, + "eval_steps_per_second": 1.362, + "step": 6000 + }, + { + "epoch": 0.39, + "learning_rate": 3.8010697906256446e-06, + "logits/chosen": -1.1743717193603516, + "logits/rejected": -0.9436849355697632, + "logps/chosen": -404.7149658203125, + "logps/rejected": -437.8125, + "loss": 0.6255, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.9801855087280273, + "rewards/margins": 0.6092766523361206, + "rewards/rejected": -2.5894620418548584, + "step": 6010 + }, + { + "epoch": 0.39, + "learning_rate": 3.7961906769308323e-06, + "logits/chosen": -0.7950822114944458, + "logits/rejected": -0.8160813450813293, + "logps/chosen": -389.021484375, + "logps/rejected": -467.6617126464844, + "loss": 0.5355, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8278629779815674, + "rewards/margins": 0.6856533288955688, + "rewards/rejected": -2.5135159492492676, + "step": 6020 + }, + { + "epoch": 0.39, + "learning_rate": 3.7913048016705028e-06, + "logits/chosen": -1.1174968481063843, + "logits/rejected": -0.9798613786697388, + "logps/chosen": -443.1385803222656, + "logps/rejected": -494.66705322265625, + "loss": 0.6275, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.8319900035858154, + "rewards/margins": 0.5357648730278015, + "rewards/rejected": -2.367755174636841, + "step": 6030 + }, + { + "epoch": 0.4, + "learning_rate": 3.786412190331775e-06, + "logits/chosen": -1.1643774509429932, + "logits/rejected": -0.7012365460395813, + "logps/chosen": -336.55694580078125, + "logps/rejected": -370.9845275878906, + "loss": 0.5543, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.3369648456573486, + "rewards/margins": 0.7152687311172485, + "rewards/rejected": -2.0522334575653076, + "step": 6040 + }, + { + "epoch": 0.4, + "learning_rate": 3.781512868436906e-06, + "logits/chosen": -1.1843137741088867, + "logits/rejected": -1.1895091533660889, + "logps/chosen": -248.87582397460938, + "logps/rejected": -328.52557373046875, + "loss": 0.4893, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.1626508235931396, + "rewards/margins": 0.7186079025268555, + "rewards/rejected": -1.8812586069107056, + "step": 6050 + }, + { + "epoch": 0.4, + "learning_rate": 3.7766068615431605e-06, + "logits/chosen": -1.017377495765686, + "logits/rejected": -0.9144327044487, + "logps/chosen": -408.78265380859375, + "logps/rejected": -416.5787048339844, + "loss": 0.5734, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.5169557332992554, + "rewards/margins": 0.5643107295036316, + "rewards/rejected": -2.081266403198242, + "step": 6060 + }, + { + "epoch": 0.4, + "learning_rate": 3.771694195242671e-06, + "logits/chosen": -1.2855536937713623, + "logits/rejected": -0.7142298817634583, + "logps/chosen": -455.6666564941406, + "logps/rejected": -439.2303771972656, + "loss": 0.4649, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.564879059791565, + "rewards/margins": 0.9529463648796082, + "rewards/rejected": -2.5178253650665283, + "step": 6070 + }, + { + "epoch": 0.4, + "learning_rate": 3.766774895162314e-06, + "logits/chosen": -0.928503155708313, + "logits/rejected": -0.9303508996963501, + "logps/chosen": -422.6429138183594, + "logps/rejected": -412.329833984375, + "loss": 0.6427, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.7610315084457397, + "rewards/margins": 0.45822644233703613, + "rewards/rejected": -2.2192580699920654, + "step": 6080 + }, + { + "epoch": 0.4, + "learning_rate": 3.7618489869635666e-06, + "logits/chosen": -0.9766615629196167, + "logits/rejected": -0.8009330630302429, + "logps/chosen": -423.50152587890625, + "logps/rejected": -477.24444580078125, + "loss": 0.6055, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7647453546524048, + "rewards/margins": 0.7348583936691284, + "rewards/rejected": -2.499603748321533, + "step": 6090 + }, + { + "epoch": 0.4, + "learning_rate": 3.756916496342379e-06, + "logits/chosen": -1.1199396848678589, + "logits/rejected": -1.1375951766967773, + "logps/chosen": -329.74737548828125, + "logps/rejected": -424.89373779296875, + "loss": 0.5633, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.4287549257278442, + "rewards/margins": 0.8669357299804688, + "rewards/rejected": -2.2956907749176025, + "step": 6100 + }, + { + "epoch": 0.4, + "eval_logits/chosen": -1.014034390449524, + "eval_logits/rejected": -0.8871217370033264, + "eval_logps/chosen": -376.6841125488281, + "eval_logps/rejected": -444.1477355957031, + "eval_loss": 0.5061184763908386, + "eval_rewards/accuracies": 0.6604999899864197, + "eval_rewards/chosen": -1.4467917680740356, + "eval_rewards/margins": 0.8785668015480042, + "eval_rewards/rejected": -2.3253581523895264, + "eval_runtime": 736.7553, + "eval_samples_per_second": 2.715, + "eval_steps_per_second": 1.357, + "step": 6100 + }, + { + "epoch": 0.4, + "learning_rate": 3.751977449029039e-06, + "logits/chosen": -0.8152976036071777, + "logits/rejected": -0.7364732623100281, + "logps/chosen": -415.014404296875, + "logps/rejected": -477.4894104003906, + "loss": 0.4901, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5493793487548828, + "rewards/margins": 0.9987069368362427, + "rewards/rejected": -2.548086643218994, + "step": 6110 + }, + { + "epoch": 0.4, + "learning_rate": 3.747031870788037e-06, + "logits/chosen": -1.1710433959960938, + "logits/rejected": -1.0110199451446533, + "logps/chosen": -455.12298583984375, + "logps/rejected": -462.619140625, + "loss": 0.4913, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.3836971521377563, + "rewards/margins": 0.8991546630859375, + "rewards/rejected": -2.2828516960144043, + "step": 6120 + }, + { + "epoch": 0.4, + "learning_rate": 3.7420797874179326e-06, + "logits/chosen": -0.899192214012146, + "logits/rejected": -0.8290202021598816, + "logps/chosen": -364.8775329589844, + "logps/rejected": -379.16156005859375, + "loss": 0.5909, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.2776801586151123, + "rewards/margins": 0.7317312359809875, + "rewards/rejected": -2.009411334991455, + "step": 6130 + }, + { + "epoch": 0.4, + "learning_rate": 3.7371212247512167e-06, + "logits/chosen": -1.3570548295974731, + "logits/rejected": -1.0728057622909546, + "logps/chosen": -430.4407653808594, + "logps/rejected": -455.63336181640625, + "loss": 0.4932, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.040462613105774, + "rewards/margins": 0.9030081629753113, + "rewards/rejected": -1.9434705972671509, + "step": 6140 + }, + { + "epoch": 0.4, + "learning_rate": 3.7321562086541817e-06, + "logits/chosen": -1.1366050243377686, + "logits/rejected": -1.0764566659927368, + "logps/chosen": -367.8480224609375, + "logps/rejected": -450.5508728027344, + "loss": 0.506, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.1486790180206299, + "rewards/margins": 0.8191258311271667, + "rewards/rejected": -1.9678049087524414, + "step": 6150 + }, + { + "epoch": 0.4, + "learning_rate": 3.7271847650267834e-06, + "logits/chosen": -0.965534508228302, + "logits/rejected": -0.8177278637886047, + "logps/chosen": -331.1851806640625, + "logps/rejected": -384.86279296875, + "loss": 0.5897, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3125053644180298, + "rewards/margins": 0.545678973197937, + "rewards/rejected": -1.8581840991973877, + "step": 6160 + }, + { + "epoch": 0.4, + "learning_rate": 3.7222069198025086e-06, + "logits/chosen": -0.8877931833267212, + "logits/rejected": -0.7562873959541321, + "logps/chosen": -348.67169189453125, + "logps/rejected": -421.09088134765625, + "loss": 0.4725, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.401292324066162, + "rewards/margins": 0.9301143884658813, + "rewards/rejected": -2.331406354904175, + "step": 6170 + }, + { + "epoch": 0.4, + "learning_rate": 3.7172226989482353e-06, + "logits/chosen": -0.8825893402099609, + "logits/rejected": -0.8212112188339233, + "logps/chosen": -349.2421875, + "logps/rejected": -415.50311279296875, + "loss": 0.571, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.392693281173706, + "rewards/margins": 0.6840615272521973, + "rewards/rejected": -2.0767548084259033, + "step": 6180 + }, + { + "epoch": 0.4, + "learning_rate": 3.7122321284641007e-06, + "logits/chosen": -1.1571756601333618, + "logits/rejected": -0.9974996447563171, + "logps/chosen": -506.38433837890625, + "logps/rejected": -488.990234375, + "loss": 0.4851, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.4359768629074097, + "rewards/margins": 0.9309576749801636, + "rewards/rejected": -2.366934299468994, + "step": 6190 + }, + { + "epoch": 0.41, + "learning_rate": 3.707235234383365e-06, + "logits/chosen": -0.8960892558097839, + "logits/rejected": -0.8411703109741211, + "logps/chosen": -375.9416198730469, + "logps/rejected": -374.1454772949219, + "loss": 0.4512, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.146876573562622, + "rewards/margins": 0.8498269319534302, + "rewards/rejected": -1.9967035055160522, + "step": 6200 + }, + { + "epoch": 0.41, + "eval_logits/chosen": -1.0898276567459106, + "eval_logits/rejected": -0.9586160182952881, + "eval_logps/chosen": -351.6016845703125, + "eval_logps/rejected": -419.078857421875, + "eval_loss": 0.5026857256889343, + "eval_rewards/accuracies": 0.6589999794960022, + "eval_rewards/chosen": -1.195967197418213, + "eval_rewards/margins": 0.8787018656730652, + "eval_rewards/rejected": -2.0746688842773438, + "eval_runtime": 732.4564, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.365, + "step": 6200 + }, + { + "epoch": 0.41, + "learning_rate": 3.702232042772277e-06, + "logits/chosen": -1.134568214416504, + "logits/rejected": -1.0888020992279053, + "logps/chosen": -348.2904968261719, + "logps/rejected": -419.7118225097656, + "loss": 0.455, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.4574432373046875, + "rewards/margins": 0.9377595782279968, + "rewards/rejected": -2.395203113555908, + "step": 6210 + }, + { + "epoch": 0.41, + "learning_rate": 3.6972225797299325e-06, + "logits/chosen": -1.1086461544036865, + "logits/rejected": -1.122828483581543, + "logps/chosen": -408.348876953125, + "logps/rejected": -480.79833984375, + "loss": 0.5259, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.5784021615982056, + "rewards/margins": 0.8838506937026978, + "rewards/rejected": -2.4622528553009033, + "step": 6220 + }, + { + "epoch": 0.41, + "learning_rate": 3.692206871388147e-06, + "logits/chosen": -1.1035563945770264, + "logits/rejected": -0.7370472550392151, + "logps/chosen": -365.0359191894531, + "logps/rejected": -439.5252990722656, + "loss": 0.414, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.3264720439910889, + "rewards/margins": 1.157185435295105, + "rewards/rejected": -2.4836573600769043, + "step": 6230 + }, + { + "epoch": 0.41, + "learning_rate": 3.6871849439113115e-06, + "logits/chosen": -0.5176571011543274, + "logits/rejected": -0.7378994226455688, + "logps/chosen": -387.4356994628906, + "logps/rejected": -446.71826171875, + "loss": 0.5859, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.5752484798431396, + "rewards/margins": 0.688303530216217, + "rewards/rejected": -2.263552188873291, + "step": 6240 + }, + { + "epoch": 0.41, + "learning_rate": 3.682156823496259e-06, + "logits/chosen": -0.99120032787323, + "logits/rejected": -0.6283798217773438, + "logps/chosen": -352.1692810058594, + "logps/rejected": -434.5711975097656, + "loss": 0.4444, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.3825536966323853, + "rewards/margins": 1.1476740837097168, + "rewards/rejected": -2.5302276611328125, + "step": 6250 + }, + { + "epoch": 0.41, + "learning_rate": 3.67712253637213e-06, + "logits/chosen": -0.9281431436538696, + "logits/rejected": -0.7577448487281799, + "logps/chosen": -429.355712890625, + "logps/rejected": -434.19647216796875, + "loss": 0.5062, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.3706127405166626, + "rewards/margins": 0.9475326538085938, + "rewards/rejected": -2.318145275115967, + "step": 6260 + }, + { + "epoch": 0.41, + "learning_rate": 3.672082108800231e-06, + "logits/chosen": -0.6212013363838196, + "logits/rejected": -0.7015591859817505, + "logps/chosen": -430.39727783203125, + "logps/rejected": -477.31982421875, + "loss": 0.5381, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.1658148765563965, + "rewards/margins": 0.8096197247505188, + "rewards/rejected": -2.9754345417022705, + "step": 6270 + }, + { + "epoch": 0.41, + "learning_rate": 3.6670355670739012e-06, + "logits/chosen": -0.7404322624206543, + "logits/rejected": -0.6079100370407104, + "logps/chosen": -349.10504150390625, + "logps/rejected": -431.7937927246094, + "loss": 0.4855, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.871516227722168, + "rewards/margins": 0.8532747030258179, + "rewards/rejected": -2.7247908115386963, + "step": 6280 + }, + { + "epoch": 0.41, + "learning_rate": 3.6619829375183745e-06, + "logits/chosen": -0.8828223347663879, + "logits/rejected": -0.7857164144515991, + "logps/chosen": -449.6969299316406, + "logps/rejected": -528.0665283203125, + "loss": 0.5199, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2369437217712402, + "rewards/margins": 0.9444055557250977, + "rewards/rejected": -3.181349515914917, + "step": 6290 + }, + { + "epoch": 0.41, + "learning_rate": 3.6569242464906427e-06, + "logits/chosen": -0.7781280279159546, + "logits/rejected": -0.7232946157455444, + "logps/chosen": -389.30474853515625, + "logps/rejected": -515.1699829101562, + "loss": 0.4765, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.815757393836975, + "rewards/margins": 1.0481603145599365, + "rewards/rejected": -2.863917350769043, + "step": 6300 + }, + { + "epoch": 0.41, + "eval_logits/chosen": -0.8425333499908447, + "eval_logits/rejected": -0.7242295145988464, + "eval_logps/chosen": -450.2899169921875, + "eval_logps/rejected": -523.9769897460938, + "eval_loss": 0.5007634162902832, + "eval_rewards/accuracies": 0.6654999852180481, + "eval_rewards/chosen": -2.182849645614624, + "eval_rewards/margins": 0.9408012628555298, + "eval_rewards/rejected": -3.1236507892608643, + "eval_runtime": 733.0976, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 6300 + }, + { + "epoch": 0.41, + "learning_rate": 3.6518595203793156e-06, + "logits/chosen": -0.8151119351387024, + "logits/rejected": -0.6735297441482544, + "logps/chosen": -491.1490173339844, + "logps/rejected": -613.0597534179688, + "loss": 0.4785, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.324347496032715, + "rewards/margins": 1.0843600034713745, + "rewards/rejected": -3.4087073802948, + "step": 6310 + }, + { + "epoch": 0.41, + "learning_rate": 3.646788785604485e-06, + "logits/chosen": -0.9957554936408997, + "logits/rejected": -1.0441629886627197, + "logps/chosen": -402.04180908203125, + "logps/rejected": -468.99884033203125, + "loss": 0.6281, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9638341665267944, + "rewards/margins": 0.692020058631897, + "rewards/rejected": -2.6558539867401123, + "step": 6320 + }, + { + "epoch": 0.41, + "learning_rate": 3.641712068617588e-06, + "logits/chosen": -0.8832969665527344, + "logits/rejected": -0.76265549659729, + "logps/chosen": -461.4491271972656, + "logps/rejected": -495.14862060546875, + "loss": 0.4246, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.0008909702301025, + "rewards/margins": 0.8724395036697388, + "rewards/rejected": -2.873330593109131, + "step": 6330 + }, + { + "epoch": 0.41, + "learning_rate": 3.6366293959012673e-06, + "logits/chosen": -0.7737056016921997, + "logits/rejected": -0.6211354732513428, + "logps/chosen": -356.0001220703125, + "logps/rejected": -422.6172790527344, + "loss": 0.4797, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.747981309890747, + "rewards/margins": 0.8614038228988647, + "rewards/rejected": -2.6093852519989014, + "step": 6340 + }, + { + "epoch": 0.42, + "learning_rate": 3.631540793969233e-06, + "logits/chosen": -1.336551308631897, + "logits/rejected": -1.1524276733398438, + "logps/chosen": -355.12652587890625, + "logps/rejected": -429.01641845703125, + "loss": 0.4994, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.6650876998901367, + "rewards/margins": 0.7221162915229797, + "rewards/rejected": -2.3872039318084717, + "step": 6350 + }, + { + "epoch": 0.42, + "learning_rate": 3.626446289366127e-06, + "logits/chosen": -1.2687801122665405, + "logits/rejected": -1.0421772003173828, + "logps/chosen": -415.6885681152344, + "logps/rejected": -421.1853942871094, + "loss": 0.5333, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9553935527801514, + "rewards/margins": 0.7701565027236938, + "rewards/rejected": -2.7255501747131348, + "step": 6360 + }, + { + "epoch": 0.42, + "learning_rate": 3.6213459086673786e-06, + "logits/chosen": -1.239116907119751, + "logits/rejected": -1.2933433055877686, + "logps/chosen": -342.5483703613281, + "logps/rejected": -441.9959411621094, + "loss": 0.5679, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7161080837249756, + "rewards/margins": 0.9007428884506226, + "rewards/rejected": -2.6168508529663086, + "step": 6370 + }, + { + "epoch": 0.42, + "learning_rate": 3.6162396784790737e-06, + "logits/chosen": -1.0483715534210205, + "logits/rejected": -0.9412117004394531, + "logps/chosen": -366.8360900878906, + "logps/rejected": -427.07275390625, + "loss": 0.5743, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.4382898807525635, + "rewards/margins": 0.6039630174636841, + "rewards/rejected": -2.042252779006958, + "step": 6380 + }, + { + "epoch": 0.42, + "learning_rate": 3.6111276254378095e-06, + "logits/chosen": -1.2586199045181274, + "logits/rejected": -1.1685311794281006, + "logps/chosen": -372.86676025390625, + "logps/rejected": -455.90179443359375, + "loss": 0.5039, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.4490987062454224, + "rewards/margins": 0.8744858503341675, + "rewards/rejected": -2.32358455657959, + "step": 6390 + }, + { + "epoch": 0.42, + "learning_rate": 3.606009776210559e-06, + "logits/chosen": -1.1810599565505981, + "logits/rejected": -1.1550124883651733, + "logps/chosen": -443.35546875, + "logps/rejected": -485.1634216308594, + "loss": 0.5056, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.027317762374878, + "rewards/margins": 0.8546678423881531, + "rewards/rejected": -2.8819851875305176, + "step": 6400 + }, + { + "epoch": 0.42, + "eval_logits/chosen": -1.1095459461212158, + "eval_logits/rejected": -0.981111466884613, + "eval_logps/chosen": -404.58245849609375, + "eval_logps/rejected": -472.86614990234375, + "eval_loss": 0.5051407217979431, + "eval_rewards/accuracies": 0.6589999794960022, + "eval_rewards/chosen": -1.7257753610610962, + "eval_rewards/margins": 0.8867669701576233, + "eval_rewards/rejected": -2.6125423908233643, + "eval_runtime": 735.605, + "eval_samples_per_second": 2.719, + "eval_steps_per_second": 1.359, + "step": 6400 + }, + { + "epoch": 0.42, + "learning_rate": 3.600886157494531e-06, + "logits/chosen": -1.3280621767044067, + "logits/rejected": -1.2239741086959839, + "logps/chosen": -441.26220703125, + "logps/rejected": -522.0778198242188, + "loss": 0.473, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7840566635131836, + "rewards/margins": 0.9259775280952454, + "rewards/rejected": -2.710033893585205, + "step": 6410 + }, + { + "epoch": 0.42, + "learning_rate": 3.5957567960170304e-06, + "logits/chosen": -1.319632887840271, + "logits/rejected": -0.8672634959220886, + "logps/chosen": -453.4800720214844, + "logps/rejected": -438.85455322265625, + "loss": 0.4254, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.6229498386383057, + "rewards/margins": 1.040321946144104, + "rewards/rejected": -2.663271427154541, + "step": 6420 + }, + { + "epoch": 0.42, + "learning_rate": 3.590621718535319e-06, + "logits/chosen": -0.9140608906745911, + "logits/rejected": -0.8867174983024597, + "logps/chosen": -370.87628173828125, + "logps/rejected": -481.146484375, + "loss": 0.5087, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7415390014648438, + "rewards/margins": 1.1116279363632202, + "rewards/rejected": -2.8531670570373535, + "step": 6430 + }, + { + "epoch": 0.42, + "learning_rate": 3.5854809518364775e-06, + "logits/chosen": -1.211717963218689, + "logits/rejected": -1.0965769290924072, + "logps/chosen": -388.0635681152344, + "logps/rejected": -460.0933532714844, + "loss": 0.4406, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.4191679954528809, + "rewards/margins": 1.1654255390167236, + "rewards/rejected": -2.5845935344696045, + "step": 6440 + }, + { + "epoch": 0.42, + "learning_rate": 3.580334522737262e-06, + "logits/chosen": -1.0971286296844482, + "logits/rejected": -1.0055302381515503, + "logps/chosen": -364.49560546875, + "logps/rejected": -415.90679931640625, + "loss": 0.5878, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6255121231079102, + "rewards/margins": 0.8506077527999878, + "rewards/rejected": -2.4761199951171875, + "step": 6450 + }, + { + "epoch": 0.42, + "learning_rate": 3.575182458083968e-06, + "logits/chosen": -1.1140817403793335, + "logits/rejected": -1.0681989192962646, + "logps/chosen": -379.82159423828125, + "logps/rejected": -475.5859375, + "loss": 0.4182, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.3901653289794922, + "rewards/margins": 1.237693190574646, + "rewards/rejected": -2.6278586387634277, + "step": 6460 + }, + { + "epoch": 0.42, + "learning_rate": 3.5700247847522883e-06, + "logits/chosen": -1.2970528602600098, + "logits/rejected": -1.2003545761108398, + "logps/chosen": -349.5565185546875, + "logps/rejected": -463.45306396484375, + "loss": 0.5227, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.4675853252410889, + "rewards/margins": 1.1391479969024658, + "rewards/rejected": -2.6067328453063965, + "step": 6470 + }, + { + "epoch": 0.42, + "learning_rate": 3.5648615296471743e-06, + "logits/chosen": -1.103219985961914, + "logits/rejected": -1.0351829528808594, + "logps/chosen": -392.39373779296875, + "logps/rejected": -552.31494140625, + "loss": 0.4326, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9503090381622314, + "rewards/margins": 1.2184739112854004, + "rewards/rejected": -3.168782949447632, + "step": 6480 + }, + { + "epoch": 0.42, + "learning_rate": 3.559692719702693e-06, + "logits/chosen": -0.9401780366897583, + "logits/rejected": -0.7638431191444397, + "logps/chosen": -493.917236328125, + "logps/rejected": -564.3541259765625, + "loss": 0.481, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.0777204036712646, + "rewards/margins": 1.2716641426086426, + "rewards/rejected": -3.349384307861328, + "step": 6490 + }, + { + "epoch": 0.43, + "learning_rate": 3.55451838188189e-06, + "logits/chosen": -1.2014122009277344, + "logits/rejected": -1.2170169353485107, + "logps/chosen": -447.1665954589844, + "logps/rejected": -580.5740356445312, + "loss": 0.5037, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8587976694107056, + "rewards/margins": 1.1147427558898926, + "rewards/rejected": -2.9735400676727295, + "step": 6500 + }, + { + "epoch": 0.43, + "eval_logits/chosen": -1.077275276184082, + "eval_logits/rejected": -0.9466902613639832, + "eval_logps/chosen": -469.41241455078125, + "eval_logps/rejected": -541.4144897460938, + "eval_loss": 0.5052820444107056, + "eval_rewards/accuracies": 0.6644999980926514, + "eval_rewards/chosen": -2.3740742206573486, + "eval_rewards/margins": 0.9239515066146851, + "eval_rewards/rejected": -3.2980258464813232, + "eval_runtime": 734.7702, + "eval_samples_per_second": 2.722, + "eval_steps_per_second": 1.361, + "step": 6500 + }, + { + "epoch": 0.43, + "learning_rate": 3.549338543176645e-06, + "logits/chosen": -1.1520912647247314, + "logits/rejected": -1.0121804475784302, + "logps/chosen": -548.9495849609375, + "logps/rejected": -615.66064453125, + "loss": 0.5094, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.4246106147766113, + "rewards/margins": 0.9906423687934875, + "rewards/rejected": -3.415252685546875, + "step": 6510 + }, + { + "epoch": 0.43, + "learning_rate": 3.5441532306075342e-06, + "logits/chosen": -1.2366782426834106, + "logits/rejected": -1.1612181663513184, + "logps/chosen": -460.84844970703125, + "logps/rejected": -562.7225952148438, + "loss": 0.6087, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.294292449951172, + "rewards/margins": 0.5530889630317688, + "rewards/rejected": -2.847381114959717, + "step": 6520 + }, + { + "epoch": 0.43, + "learning_rate": 3.5389624712236894e-06, + "logits/chosen": -1.060490369796753, + "logits/rejected": -0.873035728931427, + "logps/chosen": -434.99249267578125, + "logps/rejected": -811.9586181640625, + "loss": 0.6352, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.209566831588745, + "rewards/margins": 4.038938999176025, + "rewards/rejected": -6.248506546020508, + "step": 6530 + }, + { + "epoch": 0.43, + "learning_rate": 3.533766292102653e-06, + "logits/chosen": -0.9152925610542297, + "logits/rejected": -1.015721082687378, + "logps/chosen": -447.19976806640625, + "logps/rejected": -503.60736083984375, + "loss": 0.549, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.373551607131958, + "rewards/margins": 0.6809632182121277, + "rewards/rejected": -3.0545151233673096, + "step": 6540 + }, + { + "epoch": 0.43, + "learning_rate": 3.5285647203502404e-06, + "logits/chosen": -1.4056029319763184, + "logits/rejected": -1.2151262760162354, + "logps/chosen": -455.80096435546875, + "logps/rejected": -490.271484375, + "loss": 0.5012, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9852997064590454, + "rewards/margins": 0.7223653793334961, + "rewards/rejected": -2.707664966583252, + "step": 6550 + }, + { + "epoch": 0.43, + "learning_rate": 3.5233577831003983e-06, + "logits/chosen": -1.2194260358810425, + "logits/rejected": -1.0652081966400146, + "logps/chosen": -435.7723693847656, + "logps/rejected": -467.86370849609375, + "loss": 0.5666, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.8359639644622803, + "rewards/margins": 0.5576001405715942, + "rewards/rejected": -2.393564462661743, + "step": 6560 + }, + { + "epoch": 0.43, + "learning_rate": 3.5181455075150628e-06, + "logits/chosen": -1.1756420135498047, + "logits/rejected": -0.9271238446235657, + "logps/chosen": -322.8921203613281, + "logps/rejected": -347.01171875, + "loss": 0.5241, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.4094593524932861, + "rewards/margins": 0.7074254155158997, + "rewards/rejected": -2.116884708404541, + "step": 6570 + }, + { + "epoch": 0.43, + "learning_rate": 3.512927920784016e-06, + "logits/chosen": -1.3403713703155518, + "logits/rejected": -1.1953039169311523, + "logps/chosen": -358.7806091308594, + "logps/rejected": -436.91107177734375, + "loss": 0.4811, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.33106529712677, + "rewards/margins": 0.981104850769043, + "rewards/rejected": -2.3121702671051025, + "step": 6580 + }, + { + "epoch": 0.43, + "learning_rate": 3.5077050501247457e-06, + "logits/chosen": -1.4561182260513306, + "logits/rejected": -1.0121886730194092, + "logps/chosen": -415.90350341796875, + "logps/rejected": -451.1192321777344, + "loss": 0.4661, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3488860130310059, + "rewards/margins": 1.0260428190231323, + "rewards/rejected": -2.3749289512634277, + "step": 6590 + }, + { + "epoch": 0.43, + "learning_rate": 3.5024769227823042e-06, + "logits/chosen": -1.4400004148483276, + "logits/rejected": -1.24789559841156, + "logps/chosen": -304.3248291015625, + "logps/rejected": -340.4151916503906, + "loss": 0.5839, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.4145349264144897, + "rewards/margins": 0.7688801288604736, + "rewards/rejected": -2.183415174484253, + "step": 6600 + }, + { + "epoch": 0.43, + "eval_logits/chosen": -1.3891085386276245, + "eval_logits/rejected": -1.2408503293991089, + "eval_logps/chosen": -375.1404724121094, + "eval_logps/rejected": -446.2347412109375, + "eval_loss": 0.5008581876754761, + "eval_rewards/accuracies": 0.6710000038146973, + "eval_rewards/chosen": -1.4313548803329468, + "eval_rewards/margins": 0.914873480796814, + "eval_rewards/rejected": -2.34622859954834, + "eval_runtime": 730.2882, + "eval_samples_per_second": 2.739, + "eval_steps_per_second": 1.369, + "step": 6600 + }, + { + "epoch": 0.43, + "learning_rate": 3.4972435660291646e-06, + "logits/chosen": -1.5686732530593872, + "logits/rejected": -1.5094807147979736, + "logps/chosen": -399.5616149902344, + "logps/rejected": -459.87786865234375, + "loss": 0.5037, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5641734600067139, + "rewards/margins": 0.8525595664978027, + "rewards/rejected": -2.4167332649230957, + "step": 6610 + }, + { + "epoch": 0.43, + "learning_rate": 3.492005007165079e-06, + "logits/chosen": -1.266118049621582, + "logits/rejected": -1.173451542854309, + "logps/chosen": -376.46588134765625, + "logps/rejected": -453.89404296875, + "loss": 0.5533, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.4692888259887695, + "rewards/margins": 0.712715744972229, + "rewards/rejected": -2.182004451751709, + "step": 6620 + }, + { + "epoch": 0.43, + "learning_rate": 3.4867612735169377e-06, + "logits/chosen": -1.5591437816619873, + "logits/rejected": -1.1772428750991821, + "logps/chosen": -383.3150329589844, + "logps/rejected": -404.107666015625, + "loss": 0.459, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.5828063488006592, + "rewards/margins": 0.9902087450027466, + "rewards/rejected": -2.573014974594116, + "step": 6630 + }, + { + "epoch": 0.43, + "learning_rate": 3.4815123924386226e-06, + "logits/chosen": -1.6907329559326172, + "logits/rejected": -1.3803856372833252, + "logps/chosen": -476.39520263671875, + "logps/rejected": -472.42462158203125, + "loss": 0.5546, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.6985385417938232, + "rewards/margins": 0.6570184230804443, + "rewards/rejected": -2.3555569648742676, + "step": 6640 + }, + { + "epoch": 0.44, + "learning_rate": 3.4762583913108696e-06, + "logits/chosen": -1.1576273441314697, + "logits/rejected": -1.02208411693573, + "logps/chosen": -449.2685546875, + "logps/rejected": -480.30426025390625, + "loss": 0.61, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7951021194458008, + "rewards/margins": 0.719508707523346, + "rewards/rejected": -2.514610767364502, + "step": 6650 + }, + { + "epoch": 0.44, + "learning_rate": 3.4709992975411217e-06, + "logits/chosen": -1.2916300296783447, + "logits/rejected": -0.974404513835907, + "logps/chosen": -461.3033142089844, + "logps/rejected": -503.5069885253906, + "loss": 0.4928, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.063438892364502, + "rewards/margins": 0.9447065591812134, + "rewards/rejected": -3.008145570755005, + "step": 6660 + }, + { + "epoch": 0.44, + "learning_rate": 3.4657351385633886e-06, + "logits/chosen": -1.3369910717010498, + "logits/rejected": -1.203258752822876, + "logps/chosen": -363.0143737792969, + "logps/rejected": -436.9532165527344, + "loss": 0.4847, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7772363424301147, + "rewards/margins": 0.8613905906677246, + "rewards/rejected": -2.638627052307129, + "step": 6670 + }, + { + "epoch": 0.44, + "learning_rate": 3.4604659418381024e-06, + "logits/chosen": -1.439807653427124, + "logits/rejected": -1.0867663621902466, + "logps/chosen": -436.64892578125, + "logps/rejected": -499.2110290527344, + "loss": 0.5199, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.185945987701416, + "rewards/margins": 0.9209274053573608, + "rewards/rejected": -3.1068735122680664, + "step": 6680 + }, + { + "epoch": 0.44, + "learning_rate": 3.4551917348519744e-06, + "logits/chosen": -1.4486407041549683, + "logits/rejected": -1.2376219034194946, + "logps/chosen": -458.61328125, + "logps/rejected": -488.44256591796875, + "loss": 0.4949, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7711458206176758, + "rewards/margins": 0.770298182964325, + "rewards/rejected": -2.5414438247680664, + "step": 6690 + }, + { + "epoch": 0.44, + "learning_rate": 3.4499125451178505e-06, + "logits/chosen": -1.0467464923858643, + "logits/rejected": -1.0982073545455933, + "logps/chosen": -404.7933654785156, + "logps/rejected": -490.70892333984375, + "loss": 0.6173, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.0050206184387207, + "rewards/margins": 0.6130286455154419, + "rewards/rejected": -2.618049144744873, + "step": 6700 + }, + { + "epoch": 0.44, + "eval_logits/chosen": -1.3957716226577759, + "eval_logits/rejected": -1.2478077411651611, + "eval_logps/chosen": -415.9501953125, + "eval_logps/rejected": -482.2916259765625, + "eval_loss": 0.5004130601882935, + "eval_rewards/accuracies": 0.6694999933242798, + "eval_rewards/chosen": -1.8394523859024048, + "eval_rewards/margins": 0.8673450946807861, + "eval_rewards/rejected": -2.7067973613739014, + "eval_runtime": 733.2017, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 6700 + }, + { + "epoch": 0.44, + "learning_rate": 3.4446284001745723e-06, + "logits/chosen": -1.0857712030410767, + "logits/rejected": -0.9940765500068665, + "logps/chosen": -417.3369140625, + "logps/rejected": -508.9224548339844, + "loss": 0.5722, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.111032485961914, + "rewards/margins": 0.7483859658241272, + "rewards/rejected": -2.8594183921813965, + "step": 6710 + }, + { + "epoch": 0.44, + "learning_rate": 3.439339327586827e-06, + "logits/chosen": -1.339019536972046, + "logits/rejected": -1.3519829511642456, + "logps/chosen": -337.61767578125, + "logps/rejected": -425.98663330078125, + "loss": 0.4921, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.4759998321533203, + "rewards/margins": 0.9373260736465454, + "rewards/rejected": -2.413325786590576, + "step": 6720 + }, + { + "epoch": 0.44, + "learning_rate": 3.434045354945008e-06, + "logits/chosen": -1.425730586051941, + "logits/rejected": -1.2801324129104614, + "logps/chosen": -512.8795166015625, + "logps/rejected": -581.2686157226562, + "loss": 0.5462, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.388549327850342, + "rewards/margins": 0.7312488555908203, + "rewards/rejected": -3.119798183441162, + "step": 6730 + }, + { + "epoch": 0.44, + "learning_rate": 3.4287465098650713e-06, + "logits/chosen": -1.6729755401611328, + "logits/rejected": -1.5117638111114502, + "logps/chosen": -432.8279724121094, + "logps/rejected": -494.1000061035156, + "loss": 0.5091, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8485244512557983, + "rewards/margins": 0.7611358761787415, + "rewards/rejected": -2.6096603870391846, + "step": 6740 + }, + { + "epoch": 0.44, + "learning_rate": 3.423442819988387e-06, + "logits/chosen": -1.42362380027771, + "logits/rejected": -1.2848873138427734, + "logps/chosen": -374.7146301269531, + "logps/rejected": -472.390869140625, + "loss": 0.3997, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.9132734537124634, + "rewards/margins": 1.0486565828323364, + "rewards/rejected": -2.961930274963379, + "step": 6750 + }, + { + "epoch": 0.44, + "learning_rate": 3.4181343129816e-06, + "logits/chosen": -1.4292211532592773, + "logits/rejected": -1.2701166868209839, + "logps/chosen": -337.1582946777344, + "logps/rejected": -387.37823486328125, + "loss": 0.6429, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.6079658269882202, + "rewards/margins": 0.6451005339622498, + "rewards/rejected": -2.2530665397644043, + "step": 6760 + }, + { + "epoch": 0.44, + "learning_rate": 3.4128210165364837e-06, + "logits/chosen": -1.2692002058029175, + "logits/rejected": -1.123100996017456, + "logps/chosen": -384.36895751953125, + "logps/rejected": -498.984619140625, + "loss": 0.4937, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.92959725856781, + "rewards/margins": 1.060922384262085, + "rewards/rejected": -2.9905197620391846, + "step": 6770 + }, + { + "epoch": 0.44, + "learning_rate": 3.407502958369795e-06, + "logits/chosen": -1.3931758403778076, + "logits/rejected": -1.2321561574935913, + "logps/chosen": -426.935546875, + "logps/rejected": -491.1451110839844, + "loss": 0.5404, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.844877004623413, + "rewards/margins": 0.9342195391654968, + "rewards/rejected": -2.779096841812134, + "step": 6780 + }, + { + "epoch": 0.44, + "learning_rate": 3.4021801662231297e-06, + "logits/chosen": -1.4039561748504639, + "logits/rejected": -1.240965485572815, + "logps/chosen": -422.8265075683594, + "logps/rejected": -477.4052734375, + "loss": 0.5433, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7394969463348389, + "rewards/margins": 0.768541693687439, + "rewards/rejected": -2.5080389976501465, + "step": 6790 + }, + { + "epoch": 0.44, + "learning_rate": 3.3968526678627793e-06, + "logits/chosen": -1.2355637550354004, + "logits/rejected": -1.001617193222046, + "logps/chosen": -424.4659729003906, + "logps/rejected": -476.471923828125, + "loss": 0.4917, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5943549871444702, + "rewards/margins": 0.9610947370529175, + "rewards/rejected": -2.5554497241973877, + "step": 6800 + }, + { + "epoch": 0.44, + "eval_logits/chosen": -1.3386211395263672, + "eval_logits/rejected": -1.1951630115509033, + "eval_logps/chosen": -412.70941162109375, + "eval_logps/rejected": -478.1150207519531, + "eval_loss": 0.4987241327762604, + "eval_rewards/accuracies": 0.6669999957084656, + "eval_rewards/chosen": -1.8070447444915771, + "eval_rewards/margins": 0.8579861521720886, + "eval_rewards/rejected": -2.6650311946868896, + "eval_runtime": 730.6242, + "eval_samples_per_second": 2.737, + "eval_steps_per_second": 1.369, + "step": 6800 + }, + { + "epoch": 0.45, + "learning_rate": 3.391520491079586e-06, + "logits/chosen": -1.6253728866577148, + "logits/rejected": -1.391944408416748, + "logps/chosen": -357.5389099121094, + "logps/rejected": -396.3597717285156, + "loss": 0.5152, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6157629489898682, + "rewards/margins": 0.6935049891471863, + "rewards/rejected": -2.309267997741699, + "step": 6810 + }, + { + "epoch": 0.45, + "learning_rate": 3.3861836636887936e-06, + "logits/chosen": -1.349249005317688, + "logits/rejected": -1.1007931232452393, + "logps/chosen": -462.9054260253906, + "logps/rejected": -492.6299743652344, + "loss": 0.5324, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9608653783798218, + "rewards/margins": 0.762030303478241, + "rewards/rejected": -2.722895860671997, + "step": 6820 + }, + { + "epoch": 0.45, + "learning_rate": 3.3808422135299106e-06, + "logits/chosen": -1.238844633102417, + "logits/rejected": -1.213136911392212, + "logps/chosen": -504.2168884277344, + "logps/rejected": -639.5765380859375, + "loss": 0.4731, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1853911876678467, + "rewards/margins": 0.8228403925895691, + "rewards/rejected": -3.0082316398620605, + "step": 6830 + }, + { + "epoch": 0.45, + "learning_rate": 3.375496168466556e-06, + "logits/chosen": -1.2250182628631592, + "logits/rejected": -0.9360244870185852, + "logps/chosen": -422.16705322265625, + "logps/rejected": -448.4495544433594, + "loss": 0.5549, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.072289228439331, + "rewards/margins": 0.8220914006233215, + "rewards/rejected": -2.894380569458008, + "step": 6840 + }, + { + "epoch": 0.45, + "learning_rate": 3.3701455563863205e-06, + "logits/chosen": -1.42123544216156, + "logits/rejected": -1.2015262842178345, + "logps/chosen": -553.3240966796875, + "logps/rejected": -624.6856689453125, + "loss": 0.5084, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.6144938468933105, + "rewards/margins": 1.0768070220947266, + "rewards/rejected": -3.691300630569458, + "step": 6850 + }, + { + "epoch": 0.45, + "learning_rate": 3.3647904052006174e-06, + "logits/chosen": -1.174742341041565, + "logits/rejected": -1.0763640403747559, + "logps/chosen": -518.8255615234375, + "logps/rejected": -612.1888427734375, + "loss": 0.4688, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.5352730751037598, + "rewards/margins": 0.9035671949386597, + "rewards/rejected": -3.43884015083313, + "step": 6860 + }, + { + "epoch": 0.45, + "learning_rate": 3.3594307428445383e-06, + "logits/chosen": -1.319272756576538, + "logits/rejected": -0.9488700032234192, + "logps/chosen": -597.383544921875, + "logps/rejected": -680.8193969726562, + "loss": 0.4074, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.641327381134033, + "rewards/margins": 1.1485236883163452, + "rewards/rejected": -3.789851427078247, + "step": 6870 + }, + { + "epoch": 0.45, + "learning_rate": 3.354066597276707e-06, + "logits/chosen": -0.8110232353210449, + "logits/rejected": -0.757287323474884, + "logps/chosen": -478.07330322265625, + "logps/rejected": -609.6417236328125, + "loss": 0.517, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.5450901985168457, + "rewards/margins": 0.9620795249938965, + "rewards/rejected": -3.507169723510742, + "step": 6880 + }, + { + "epoch": 0.45, + "learning_rate": 3.348697996479136e-06, + "logits/chosen": -0.9675430059432983, + "logits/rejected": -0.79949551820755, + "logps/chosen": -529.7903442382812, + "logps/rejected": -559.5513305664062, + "loss": 0.5838, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -3.0198137760162354, + "rewards/margins": 0.8323270082473755, + "rewards/rejected": -3.8521409034729004, + "step": 6890 + }, + { + "epoch": 0.45, + "learning_rate": 3.3433249684570757e-06, + "logits/chosen": -0.7183124423027039, + "logits/rejected": -0.524259090423584, + "logps/chosen": -416.25115966796875, + "logps/rejected": -479.09478759765625, + "loss": 0.4834, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.384216070175171, + "rewards/margins": 1.0856199264526367, + "rewards/rejected": -3.469836473464966, + "step": 6900 + }, + { + "epoch": 0.45, + "eval_logits/chosen": -0.9490465521812439, + "eval_logits/rejected": -0.8229533433914185, + "eval_logps/chosen": -473.6738586425781, + "eval_logps/rejected": -550.5955200195312, + "eval_loss": 0.4964263439178467, + "eval_rewards/accuracies": 0.6679999828338623, + "eval_rewards/chosen": -2.4166886806488037, + "eval_rewards/margins": 0.9731472730636597, + "eval_rewards/rejected": -3.389836072921753, + "eval_runtime": 733.502, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.363, + "step": 6900 + }, + { + "epoch": 0.45, + "learning_rate": 3.3379475412388724e-06, + "logits/chosen": -1.0790557861328125, + "logits/rejected": -0.9136208295822144, + "logps/chosen": -475.306640625, + "logps/rejected": -564.6021728515625, + "loss": 0.3662, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.3335165977478027, + "rewards/margins": 1.22994065284729, + "rewards/rejected": -3.563457489013672, + "step": 6910 + }, + { + "epoch": 0.45, + "learning_rate": 3.3325657428758207e-06, + "logits/chosen": -0.9100483059883118, + "logits/rejected": -0.9243875741958618, + "logps/chosen": -459.96026611328125, + "logps/rejected": -571.3048095703125, + "loss": 0.5337, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1628475189208984, + "rewards/margins": 1.1051790714263916, + "rewards/rejected": -3.268026351928711, + "step": 6920 + }, + { + "epoch": 0.45, + "learning_rate": 3.3271796014420175e-06, + "logits/chosen": -1.1402623653411865, + "logits/rejected": -0.8654245138168335, + "logps/chosen": -442.6941833496094, + "logps/rejected": -542.1939697265625, + "loss": 0.4406, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.302438735961914, + "rewards/margins": 1.1720813512802124, + "rewards/rejected": -3.474520206451416, + "step": 6930 + }, + { + "epoch": 0.45, + "learning_rate": 3.3217891450342142e-06, + "logits/chosen": -0.9778724908828735, + "logits/rejected": -0.8319625854492188, + "logps/chosen": -456.13482666015625, + "logps/rejected": -527.78759765625, + "loss": 0.4142, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9753639698028564, + "rewards/margins": 1.4064559936523438, + "rewards/rejected": -3.3818199634552, + "step": 6940 + }, + { + "epoch": 0.45, + "learning_rate": 3.3163944017716733e-06, + "logits/chosen": -1.2695777416229248, + "logits/rejected": -1.0632556676864624, + "logps/chosen": -419.833740234375, + "logps/rejected": -480.20281982421875, + "loss": 0.5368, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.0346288681030273, + "rewards/margins": 0.9346679449081421, + "rewards/rejected": -2.96929669380188, + "step": 6950 + }, + { + "epoch": 0.46, + "learning_rate": 3.310995399796017e-06, + "logits/chosen": -1.276155710220337, + "logits/rejected": -1.2048354148864746, + "logps/chosen": -510.0116271972656, + "logps/rejected": -541.8460083007812, + "loss": 0.8078, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.3602535724639893, + "rewards/margins": 0.38636505603790283, + "rewards/rejected": -2.7466189861297607, + "step": 6960 + }, + { + "epoch": 0.46, + "learning_rate": 3.305592167271085e-06, + "logits/chosen": -1.2273608446121216, + "logits/rejected": -1.0127642154693604, + "logps/chosen": -371.83209228515625, + "logps/rejected": -457.48052978515625, + "loss": 0.4819, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.745859146118164, + "rewards/margins": 0.9656050801277161, + "rewards/rejected": -2.7114641666412354, + "step": 6970 + }, + { + "epoch": 0.46, + "learning_rate": 3.3001847323827846e-06, + "logits/chosen": -1.2883787155151367, + "logits/rejected": -1.3649321794509888, + "logps/chosen": -454.60198974609375, + "logps/rejected": -536.0913696289062, + "loss": 0.5006, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8650429248809814, + "rewards/margins": 0.8490816354751587, + "rewards/rejected": -2.7141242027282715, + "step": 6980 + }, + { + "epoch": 0.46, + "learning_rate": 3.2947731233389447e-06, + "logits/chosen": -1.218878984451294, + "logits/rejected": -0.999015212059021, + "logps/chosen": -427.6170959472656, + "logps/rejected": -480.9115295410156, + "loss": 0.4415, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.801103949546814, + "rewards/margins": 1.0872182846069336, + "rewards/rejected": -2.888322353363037, + "step": 6990 + }, + { + "epoch": 0.46, + "learning_rate": 3.2893573683691706e-06, + "logits/chosen": -1.1505297422409058, + "logits/rejected": -1.0928208827972412, + "logps/chosen": -368.4134521484375, + "logps/rejected": -443.3638610839844, + "loss": 0.4668, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.6371997594833374, + "rewards/margins": 0.9284623861312866, + "rewards/rejected": -2.565662384033203, + "step": 7000 + }, + { + "epoch": 0.46, + "eval_logits/chosen": -1.265947699546814, + "eval_logits/rejected": -1.1271545886993408, + "eval_logps/chosen": -399.3541259765625, + "eval_logps/rejected": -466.1047058105469, + "eval_loss": 0.50328129529953, + "eval_rewards/accuracies": 0.6700000166893005, + "eval_rewards/chosen": -1.6734919548034668, + "eval_rewards/margins": 0.8714357614517212, + "eval_rewards/rejected": -2.5449280738830566, + "eval_runtime": 733.2514, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 7000 + }, + { + "epoch": 0.46, + "learning_rate": 3.2839374957246915e-06, + "logits/chosen": -1.3625187873840332, + "logits/rejected": -1.1006382703781128, + "logps/chosen": -453.134033203125, + "logps/rejected": -413.80291748046875, + "loss": 0.6148, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.8704373836517334, + "rewards/margins": 0.5220705270767212, + "rewards/rejected": -2.392508029937744, + "step": 7010 + }, + { + "epoch": 0.46, + "learning_rate": 3.2785135336782187e-06, + "logits/chosen": -1.184256911277771, + "logits/rejected": -1.0579241514205933, + "logps/chosen": -420.9827575683594, + "logps/rejected": -543.4996948242188, + "loss": 0.4591, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8706735372543335, + "rewards/margins": 1.0020452737808228, + "rewards/rejected": -2.872718572616577, + "step": 7020 + }, + { + "epoch": 0.46, + "learning_rate": 3.2730855105237952e-06, + "logits/chosen": -1.2827327251434326, + "logits/rejected": -1.1696199178695679, + "logps/chosen": -378.5633544921875, + "logps/rejected": -528.2980346679688, + "loss": 0.4283, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5828688144683838, + "rewards/margins": 1.055541753768921, + "rewards/rejected": -2.638411045074463, + "step": 7030 + }, + { + "epoch": 0.46, + "learning_rate": 3.2676534545766486e-06, + "logits/chosen": -1.1060223579406738, + "logits/rejected": -0.9897655248641968, + "logps/chosen": -374.9141540527344, + "logps/rejected": -429.4541015625, + "loss": 0.5823, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.624446153640747, + "rewards/margins": 0.6443280577659607, + "rewards/rejected": -2.2687745094299316, + "step": 7040 + }, + { + "epoch": 0.46, + "learning_rate": 3.262217394173043e-06, + "logits/chosen": -1.0637996196746826, + "logits/rejected": -0.9355257749557495, + "logps/chosen": -403.6563415527344, + "logps/rejected": -503.54949951171875, + "loss": 0.5271, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.6280568838119507, + "rewards/margins": 1.0597625970840454, + "rewards/rejected": -2.687819719314575, + "step": 7050 + }, + { + "epoch": 0.46, + "learning_rate": 3.2567773576701333e-06, + "logits/chosen": -1.0611984729766846, + "logits/rejected": -0.9134718179702759, + "logps/chosen": -395.8488464355469, + "logps/rejected": -480.60406494140625, + "loss": 0.4829, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.4009135961532593, + "rewards/margins": 1.1507080793380737, + "rewards/rejected": -2.551621675491333, + "step": 7060 + }, + { + "epoch": 0.46, + "learning_rate": 3.2513333734458154e-06, + "logits/chosen": -1.145353078842163, + "logits/rejected": -0.9989140629768372, + "logps/chosen": -377.0892639160156, + "logps/rejected": -417.6884765625, + "loss": 0.5898, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.702479600906372, + "rewards/margins": 0.5720685720443726, + "rewards/rejected": -2.274547815322876, + "step": 7070 + }, + { + "epoch": 0.46, + "learning_rate": 3.245885469898576e-06, + "logits/chosen": -1.0995618104934692, + "logits/rejected": -1.0788781642913818, + "logps/chosen": -447.03076171875, + "logps/rejected": -490.30157470703125, + "loss": 0.4897, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.464755892753601, + "rewards/margins": 1.069588303565979, + "rewards/rejected": -2.534344434738159, + "step": 7080 + }, + { + "epoch": 0.46, + "learning_rate": 3.2404336754473497e-06, + "logits/chosen": -1.1354840993881226, + "logits/rejected": -0.9494547843933105, + "logps/chosen": -395.81268310546875, + "logps/rejected": -423.93695068359375, + "loss": 0.4502, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.2830572128295898, + "rewards/margins": 0.9289452433586121, + "rewards/rejected": -2.2120022773742676, + "step": 7090 + }, + { + "epoch": 0.46, + "learning_rate": 3.234978018531367e-06, + "logits/chosen": -1.5512938499450684, + "logits/rejected": -1.0924882888793945, + "logps/chosen": -404.32403564453125, + "logps/rejected": -448.44610595703125, + "loss": 0.4544, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.458237886428833, + "rewards/margins": 1.089217185974121, + "rewards/rejected": -2.547455310821533, + "step": 7100 + }, + { + "epoch": 0.46, + "eval_logits/chosen": -1.0685359239578247, + "eval_logits/rejected": -0.9393064379692078, + "eval_logps/chosen": -391.1265563964844, + "eval_logps/rejected": -470.7079772949219, + "eval_loss": 0.4963315725326538, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -1.591215968132019, + "eval_rewards/margins": 0.9997445940971375, + "eval_rewards/rejected": -2.5909605026245117, + "eval_runtime": 730.6695, + "eval_samples_per_second": 2.737, + "eval_steps_per_second": 1.369, + "step": 7100 + }, + { + "epoch": 0.47, + "learning_rate": 3.229518527610006e-06, + "logits/chosen": -1.2886658906936646, + "logits/rejected": -1.1227843761444092, + "logps/chosen": -484.5293884277344, + "logps/rejected": -517.5733642578125, + "loss": 0.5212, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9173930883407593, + "rewards/margins": 0.7821990251541138, + "rewards/rejected": -2.699592113494873, + "step": 7110 + }, + { + "epoch": 0.47, + "learning_rate": 3.2240552311626465e-06, + "logits/chosen": -1.097480058670044, + "logits/rejected": -0.9205149412155151, + "logps/chosen": -396.7735900878906, + "logps/rejected": -465.60107421875, + "loss": 0.5388, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.5555986166000366, + "rewards/margins": 0.837724506855011, + "rewards/rejected": -2.3933229446411133, + "step": 7120 + }, + { + "epoch": 0.47, + "learning_rate": 3.2185881576885193e-06, + "logits/chosen": -1.0562323331832886, + "logits/rejected": -0.9030343890190125, + "logps/chosen": -423.342041015625, + "logps/rejected": -465.4990234375, + "loss": 0.5439, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.116192102432251, + "rewards/margins": 0.8082444071769714, + "rewards/rejected": -2.924436569213867, + "step": 7130 + }, + { + "epoch": 0.47, + "learning_rate": 3.213117335706557e-06, + "logits/chosen": -1.1086636781692505, + "logits/rejected": -1.045616865158081, + "logps/chosen": -455.88922119140625, + "logps/rejected": -570.3858642578125, + "loss": 0.5255, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9984424114227295, + "rewards/margins": 1.0529043674468994, + "rewards/rejected": -3.05134654045105, + "step": 7140 + }, + { + "epoch": 0.47, + "learning_rate": 3.2076427937552473e-06, + "logits/chosen": -1.0971708297729492, + "logits/rejected": -0.7860563397407532, + "logps/chosen": -406.85443115234375, + "logps/rejected": -514.7252197265625, + "loss": 0.4371, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6136468648910522, + "rewards/margins": 1.2699447870254517, + "rewards/rejected": -2.883591651916504, + "step": 7150 + }, + { + "epoch": 0.47, + "learning_rate": 3.2021645603924827e-06, + "logits/chosen": -0.8555929064750671, + "logits/rejected": -0.8638184666633606, + "logps/chosen": -283.649658203125, + "logps/rejected": -406.4692077636719, + "loss": 0.5197, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.4519553184509277, + "rewards/margins": 1.1345512866973877, + "rewards/rejected": -2.5865063667297363, + "step": 7160 + }, + { + "epoch": 0.47, + "learning_rate": 3.196682664195412e-06, + "logits/chosen": -0.9364141225814819, + "logits/rejected": -0.8269764184951782, + "logps/chosen": -361.3995056152344, + "logps/rejected": -384.99273681640625, + "loss": 0.6921, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.5735127925872803, + "rewards/margins": 0.5848428010940552, + "rewards/rejected": -2.158355712890625, + "step": 7170 + }, + { + "epoch": 0.47, + "learning_rate": 3.191197133760291e-06, + "logits/chosen": -1.6076021194458008, + "logits/rejected": -1.082669973373413, + "logps/chosen": -394.1782531738281, + "logps/rejected": -455.48870849609375, + "loss": 0.424, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.2677477598190308, + "rewards/margins": 1.3555927276611328, + "rewards/rejected": -2.623340606689453, + "step": 7180 + }, + { + "epoch": 0.47, + "learning_rate": 3.185707997702334e-06, + "logits/chosen": -1.2360800504684448, + "logits/rejected": -0.9020748138427734, + "logps/chosen": -411.65325927734375, + "logps/rejected": -461.48162841796875, + "loss": 0.5452, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7088842391967773, + "rewards/margins": 0.8767988085746765, + "rewards/rejected": -2.5856828689575195, + "step": 7190 + }, + { + "epoch": 0.47, + "learning_rate": 3.1802152846555624e-06, + "logits/chosen": -1.1463711261749268, + "logits/rejected": -0.976833701133728, + "logps/chosen": -398.6136474609375, + "logps/rejected": -483.20458984375, + "loss": 0.5048, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.743706464767456, + "rewards/margins": 1.001102328300476, + "rewards/rejected": -2.7448086738586426, + "step": 7200 + }, + { + "epoch": 0.47, + "eval_logits/chosen": -1.1263185739517212, + "eval_logits/rejected": -0.9988316297531128, + "eval_logps/chosen": -396.180419921875, + "eval_logps/rejected": -459.2229309082031, + "eval_loss": 0.5000500679016113, + "eval_rewards/accuracies": 0.6675000190734863, + "eval_rewards/chosen": -1.641754150390625, + "eval_rewards/margins": 0.8343563079833984, + "eval_rewards/rejected": -2.4761104583740234, + "eval_runtime": 731.8554, + "eval_samples_per_second": 2.733, + "eval_steps_per_second": 1.366, + "step": 7200 + }, + { + "epoch": 0.47, + "learning_rate": 3.174719023272659e-06, + "logits/chosen": -1.390661597251892, + "logits/rejected": -1.3155701160430908, + "logps/chosen": -373.8555908203125, + "logps/rejected": -508.87786865234375, + "loss": 0.5011, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6016145944595337, + "rewards/margins": 0.8867307901382446, + "rewards/rejected": -2.4883456230163574, + "step": 7210 + }, + { + "epoch": 0.47, + "learning_rate": 3.169219242224816e-06, + "logits/chosen": -1.3606770038604736, + "logits/rejected": -1.1562891006469727, + "logps/chosen": -411.99658203125, + "logps/rejected": -466.5486755371094, + "loss": 0.611, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7206933498382568, + "rewards/margins": 0.6064735651016235, + "rewards/rejected": -2.327166795730591, + "step": 7220 + }, + { + "epoch": 0.47, + "learning_rate": 3.1637159702015837e-06, + "logits/chosen": -1.2122749090194702, + "logits/rejected": -1.0905072689056396, + "logps/chosen": -350.120849609375, + "logps/rejected": -435.25323486328125, + "loss": 0.4162, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.4422870874404907, + "rewards/margins": 1.0445516109466553, + "rewards/rejected": -2.4868385791778564, + "step": 7230 + }, + { + "epoch": 0.47, + "learning_rate": 3.1582092359107263e-06, + "logits/chosen": -1.2009532451629639, + "logits/rejected": -0.9746109843254089, + "logps/chosen": -454.6007385253906, + "logps/rejected": -489.78302001953125, + "loss": 0.5496, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.763230323791504, + "rewards/margins": 0.8101302981376648, + "rewards/rejected": -2.5733604431152344, + "step": 7240 + }, + { + "epoch": 0.47, + "learning_rate": 3.152699068078067e-06, + "logits/chosen": -1.0341651439666748, + "logits/rejected": -0.9404195547103882, + "logps/chosen": -481.7880859375, + "logps/rejected": -568.8102416992188, + "loss": 0.4662, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0149779319763184, + "rewards/margins": 1.0756332874298096, + "rewards/rejected": -3.090610980987549, + "step": 7250 + }, + { + "epoch": 0.48, + "learning_rate": 3.1471854954473415e-06, + "logits/chosen": -1.2517516613006592, + "logits/rejected": -1.1824498176574707, + "logps/chosen": -404.09130859375, + "logps/rejected": -505.6543884277344, + "loss": 0.4441, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5332467555999756, + "rewards/margins": 1.053310513496399, + "rewards/rejected": -2.586557149887085, + "step": 7260 + }, + { + "epoch": 0.48, + "learning_rate": 3.1416685467800436e-06, + "logits/chosen": -0.916897177696228, + "logits/rejected": -0.5630122423171997, + "logps/chosen": -382.2656555175781, + "logps/rejected": -469.0830078125, + "loss": 0.4253, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9668285846710205, + "rewards/margins": 1.0934703350067139, + "rewards/rejected": -3.0602986812591553, + "step": 7270 + }, + { + "epoch": 0.48, + "learning_rate": 3.1361482508552803e-06, + "logits/chosen": -1.0097547769546509, + "logits/rejected": -0.8572899699211121, + "logps/chosen": -464.41973876953125, + "logps/rejected": -529.024658203125, + "loss": 0.4495, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2246882915496826, + "rewards/margins": 1.0799113512039185, + "rewards/rejected": -3.3045997619628906, + "step": 7280 + }, + { + "epoch": 0.48, + "learning_rate": 3.1306246364696198e-06, + "logits/chosen": -1.2491976022720337, + "logits/rejected": -1.0045207738876343, + "logps/chosen": -477.87872314453125, + "logps/rejected": -540.9088134765625, + "loss": 0.5529, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.2540440559387207, + "rewards/margins": 0.8053514361381531, + "rewards/rejected": -3.0593953132629395, + "step": 7290 + }, + { + "epoch": 0.48, + "learning_rate": 3.1250977324369413e-06, + "logits/chosen": -0.8631597757339478, + "logits/rejected": -0.7546287775039673, + "logps/chosen": -353.9215393066406, + "logps/rejected": -460.38671875, + "loss": 0.5141, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.949690818786621, + "rewards/margins": 0.9975317120552063, + "rewards/rejected": -2.9472222328186035, + "step": 7300 + }, + { + "epoch": 0.48, + "eval_logits/chosen": -0.9431403875350952, + "eval_logits/rejected": -0.8168740272521973, + "eval_logps/chosen": -440.5569763183594, + "eval_logps/rejected": -534.328125, + "eval_loss": 0.4976823031902313, + "eval_rewards/accuracies": 0.6679999828338623, + "eval_rewards/chosen": -2.0855202674865723, + "eval_rewards/margins": 1.1416418552398682, + "eval_rewards/rejected": -3.2271616458892822, + "eval_runtime": 734.1583, + "eval_samples_per_second": 2.724, + "eval_steps_per_second": 1.362, + "step": 7300 + }, + { + "epoch": 0.48, + "learning_rate": 3.1195675675882825e-06, + "logits/chosen": -1.0076789855957031, + "logits/rejected": -0.7726067900657654, + "logps/chosen": -480.07440185546875, + "logps/rejected": -547.6993408203125, + "loss": 0.6865, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.44883394241333, + "rewards/margins": 0.9822977185249329, + "rewards/rejected": -3.431131362915039, + "step": 7310 + }, + { + "epoch": 0.48, + "learning_rate": 3.1140341707716926e-06, + "logits/chosen": -0.6309999227523804, + "logits/rejected": -0.5980799794197083, + "logps/chosen": -380.33355712890625, + "logps/rejected": -449.6524353027344, + "loss": 0.5185, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8452589511871338, + "rewards/margins": 1.123349905014038, + "rewards/rejected": -2.9686086177825928, + "step": 7320 + }, + { + "epoch": 0.48, + "learning_rate": 3.1084975708520803e-06, + "logits/chosen": -1.2348999977111816, + "logits/rejected": -0.9366706013679504, + "logps/chosen": -439.20257568359375, + "logps/rejected": -488.87420654296875, + "loss": 0.3986, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.7747869491577148, + "rewards/margins": 1.2041819095611572, + "rewards/rejected": -2.978969097137451, + "step": 7330 + }, + { + "epoch": 0.48, + "learning_rate": 3.1029577967110625e-06, + "logits/chosen": -1.1992151737213135, + "logits/rejected": -0.9312963485717773, + "logps/chosen": -398.64178466796875, + "logps/rejected": -415.91705322265625, + "loss": 0.5562, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.8224748373031616, + "rewards/margins": 0.6633359789848328, + "rewards/rejected": -2.4858107566833496, + "step": 7340 + }, + { + "epoch": 0.48, + "learning_rate": 3.097414877246814e-06, + "logits/chosen": -1.1034296751022339, + "logits/rejected": -0.8228904008865356, + "logps/chosen": -375.7052307128906, + "logps/rejected": -454.7461853027344, + "loss": 0.5736, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.7706174850463867, + "rewards/margins": 1.085538387298584, + "rewards/rejected": -2.8561558723449707, + "step": 7350 + }, + { + "epoch": 0.48, + "learning_rate": 3.0918688413739197e-06, + "logits/chosen": -1.0494706630706787, + "logits/rejected": -0.7607582211494446, + "logps/chosen": -391.17401123046875, + "logps/rejected": -433.62841796875, + "loss": 0.5225, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.5849109888076782, + "rewards/margins": 1.0672708749771118, + "rewards/rejected": -2.65218186378479, + "step": 7360 + }, + { + "epoch": 0.48, + "learning_rate": 3.0863197180232178e-06, + "logits/chosen": -1.0603563785552979, + "logits/rejected": -0.9168928861618042, + "logps/chosen": -383.76544189453125, + "logps/rejected": -453.3426818847656, + "loss": 0.5822, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8402124643325806, + "rewards/margins": 0.8179795145988464, + "rewards/rejected": -2.6581919193267822, + "step": 7370 + }, + { + "epoch": 0.48, + "learning_rate": 3.0807675361416554e-06, + "logits/chosen": -0.9228275418281555, + "logits/rejected": -0.8080077171325684, + "logps/chosen": -340.25274658203125, + "logps/rejected": -358.236083984375, + "loss": 0.4654, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.513724684715271, + "rewards/margins": 0.9701772928237915, + "rewards/rejected": -2.4839017391204834, + "step": 7380 + }, + { + "epoch": 0.48, + "learning_rate": 3.0752123246921327e-06, + "logits/chosen": -1.0758297443389893, + "logits/rejected": -0.7389814853668213, + "logps/chosen": -491.60467529296875, + "logps/rejected": -515.2469482421875, + "loss": 0.5163, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.1369142532348633, + "rewards/margins": 0.9057363271713257, + "rewards/rejected": -3.0426506996154785, + "step": 7390 + }, + { + "epoch": 0.48, + "learning_rate": 3.069654112653353e-06, + "logits/chosen": -1.2086726427078247, + "logits/rejected": -0.9895984530448914, + "logps/chosen": -429.166259765625, + "logps/rejected": -445.71527099609375, + "loss": 0.646, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.1268205642700195, + "rewards/margins": 0.49101099371910095, + "rewards/rejected": -2.6178317070007324, + "step": 7400 + }, + { + "epoch": 0.48, + "eval_logits/chosen": -1.0571123361587524, + "eval_logits/rejected": -0.9287133812904358, + "eval_logps/chosen": -424.531494140625, + "eval_logps/rejected": -497.04150390625, + "eval_loss": 0.49757885932922363, + "eval_rewards/accuracies": 0.6679999828338623, + "eval_rewards/chosen": -1.9252653121948242, + "eval_rewards/margins": 0.9290300011634827, + "eval_rewards/rejected": -2.854295492172241, + "eval_runtime": 732.9108, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.364, + "step": 7400 + }, + { + "epoch": 0.48, + "learning_rate": 3.064092929019673e-06, + "logits/chosen": -1.0198802947998047, + "logits/rejected": -1.211188554763794, + "logps/chosen": -459.1338806152344, + "logps/rejected": -546.1898193359375, + "loss": 0.5362, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.015903949737549, + "rewards/margins": 0.6718121767044067, + "rewards/rejected": -2.687716007232666, + "step": 7410 + }, + { + "epoch": 0.49, + "learning_rate": 3.058528802800952e-06, + "logits/chosen": -1.2751950025558472, + "logits/rejected": -1.0569217205047607, + "logps/chosen": -469.09527587890625, + "logps/rejected": -543.2975463867188, + "loss": 0.4846, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7646152973175049, + "rewards/margins": 1.107751727104187, + "rewards/rejected": -2.8723669052124023, + "step": 7420 + }, + { + "epoch": 0.49, + "learning_rate": 3.052961763022397e-06, + "logits/chosen": -1.4869779348373413, + "logits/rejected": -0.9571384191513062, + "logps/chosen": -338.5403747558594, + "logps/rejected": -422.4949645996094, + "loss": 0.4403, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5565922260284424, + "rewards/margins": 1.2267820835113525, + "rewards/rejected": -2.783374309539795, + "step": 7430 + }, + { + "epoch": 0.49, + "learning_rate": 3.047391838724415e-06, + "logits/chosen": -1.393455147743225, + "logits/rejected": -1.1742329597473145, + "logps/chosen": -411.403564453125, + "logps/rejected": -498.971435546875, + "loss": 0.4829, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7588484287261963, + "rewards/margins": 1.0363447666168213, + "rewards/rejected": -2.7951929569244385, + "step": 7440 + }, + { + "epoch": 0.49, + "learning_rate": 3.0418190589624587e-06, + "logits/chosen": -1.0927526950836182, + "logits/rejected": -0.9543555378913879, + "logps/chosen": -354.21954345703125, + "logps/rejected": -436.81353759765625, + "loss": 0.5101, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.7659744024276733, + "rewards/margins": 0.742416262626648, + "rewards/rejected": -2.5083906650543213, + "step": 7450 + }, + { + "epoch": 0.49, + "learning_rate": 3.0362434528068784e-06, + "logits/chosen": -0.9671896696090698, + "logits/rejected": -0.7721429467201233, + "logps/chosen": -487.379150390625, + "logps/rejected": -489.2608337402344, + "loss": 0.5641, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.1805596351623535, + "rewards/margins": 0.8737584948539734, + "rewards/rejected": -3.0543181896209717, + "step": 7460 + }, + { + "epoch": 0.49, + "learning_rate": 3.0306650493427657e-06, + "logits/chosen": -1.1028976440429688, + "logits/rejected": -0.9604462385177612, + "logps/chosen": -431.97442626953125, + "logps/rejected": -527.2906494140625, + "loss": 0.4603, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.003551959991455, + "rewards/margins": 1.0427815914154053, + "rewards/rejected": -3.0463337898254395, + "step": 7470 + }, + { + "epoch": 0.49, + "learning_rate": 3.0250838776698077e-06, + "logits/chosen": -1.2236485481262207, + "logits/rejected": -1.0227609872817993, + "logps/chosen": -366.2548828125, + "logps/rejected": -462.3900451660156, + "loss": 0.4685, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8233562707901, + "rewards/margins": 0.9888140559196472, + "rewards/rejected": -2.8121702671051025, + "step": 7480 + }, + { + "epoch": 0.49, + "learning_rate": 3.0194999669021275e-06, + "logits/chosen": -0.9461000561714172, + "logits/rejected": -0.660954475402832, + "logps/chosen": -390.1704406738281, + "logps/rejected": -483.80126953125, + "loss": 0.3723, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6330198049545288, + "rewards/margins": 1.4068280458450317, + "rewards/rejected": -3.0398478507995605, + "step": 7490 + }, + { + "epoch": 0.49, + "learning_rate": 3.0139133461681403e-06, + "logits/chosen": -1.2607884407043457, + "logits/rejected": -1.0787745714187622, + "logps/chosen": -435.36334228515625, + "logps/rejected": -533.2015991210938, + "loss": 0.3417, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.709691047668457, + "rewards/margins": 1.5397835969924927, + "rewards/rejected": -3.249474287033081, + "step": 7500 + }, + { + "epoch": 0.49, + "eval_logits/chosen": -1.1435625553131104, + "eval_logits/rejected": -1.0097819566726685, + "eval_logps/chosen": -411.1138610839844, + "eval_logps/rejected": -493.5840148925781, + "eval_loss": 0.493692010641098, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -1.7910891771316528, + "eval_rewards/margins": 1.0286314487457275, + "eval_rewards/rejected": -2.81972074508667, + "eval_runtime": 733.7776, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 7500 + }, + { + "epoch": 0.49, + "learning_rate": 3.0083240446103965e-06, + "logits/chosen": -0.9401206970214844, + "logits/rejected": -0.7115296125411987, + "logps/chosen": -353.5488586425781, + "logps/rejected": -463.7230529785156, + "loss": 0.4751, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.695142388343811, + "rewards/margins": 1.0250508785247803, + "rewards/rejected": -2.720193386077881, + "step": 7510 + }, + { + "epoch": 0.49, + "learning_rate": 3.0027320913854306e-06, + "logits/chosen": -1.4791367053985596, + "logits/rejected": -1.2607471942901611, + "logps/chosen": -449.88909912109375, + "logps/rejected": -552.6820678710938, + "loss": 0.4171, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.5742747783660889, + "rewards/margins": 1.6550228595733643, + "rewards/rejected": -3.229297637939453, + "step": 7520 + }, + { + "epoch": 0.49, + "learning_rate": 2.997137515663609e-06, + "logits/chosen": -1.2254869937896729, + "logits/rejected": -1.0456231832504272, + "logps/chosen": -365.78118896484375, + "logps/rejected": -470.77374267578125, + "loss": 0.4353, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.4040377140045166, + "rewards/margins": 1.426917314529419, + "rewards/rejected": -2.8309547901153564, + "step": 7530 + }, + { + "epoch": 0.49, + "learning_rate": 2.991540346628981e-06, + "logits/chosen": -1.089773178100586, + "logits/rejected": -1.074831247329712, + "logps/chosen": -462.69830322265625, + "logps/rejected": -528.5394287109375, + "loss": 0.6096, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.2309799194335938, + "rewards/margins": 0.9194940328598022, + "rewards/rejected": -3.1504738330841064, + "step": 7540 + }, + { + "epoch": 0.49, + "learning_rate": 2.985940613479121e-06, + "logits/chosen": -1.2983834743499756, + "logits/rejected": -1.1657660007476807, + "logps/chosen": -481.27520751953125, + "logps/rejected": -534.2381591796875, + "loss": 0.4576, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8742554187774658, + "rewards/margins": 1.1233479976654053, + "rewards/rejected": -2.997603178024292, + "step": 7550 + }, + { + "epoch": 0.49, + "learning_rate": 2.980338345424981e-06, + "logits/chosen": -1.0275734663009644, + "logits/rejected": -0.9290380477905273, + "logps/chosen": -459.503662109375, + "logps/rejected": -502.5309143066406, + "loss": 0.4863, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.1028387546539307, + "rewards/margins": 0.9130401611328125, + "rewards/rejected": -3.0158791542053223, + "step": 7560 + }, + { + "epoch": 0.5, + "learning_rate": 2.974733571690735e-06, + "logits/chosen": -1.2624428272247314, + "logits/rejected": -0.8733476400375366, + "logps/chosen": -439.00433349609375, + "logps/rejected": -475.9070739746094, + "loss": 0.5729, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.015974998474121, + "rewards/margins": 0.961951732635498, + "rewards/rejected": -2.977926731109619, + "step": 7570 + }, + { + "epoch": 0.5, + "learning_rate": 2.9691263215136274e-06, + "logits/chosen": -1.3705028295516968, + "logits/rejected": -1.253151535987854, + "logps/chosen": -424.6815490722656, + "logps/rejected": -504.9693298339844, + "loss": 0.4176, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.576811671257019, + "rewards/margins": 1.1194231510162354, + "rewards/rejected": -2.696234941482544, + "step": 7580 + }, + { + "epoch": 0.5, + "learning_rate": 2.963516624143823e-06, + "logits/chosen": -1.0707788467407227, + "logits/rejected": -1.2443617582321167, + "logps/chosen": -433.29901123046875, + "logps/rejected": -480.945556640625, + "loss": 0.5844, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.2073233127593994, + "rewards/margins": 0.8517726063728333, + "rewards/rejected": -3.059095859527588, + "step": 7590 + }, + { + "epoch": 0.5, + "learning_rate": 2.9579045088442504e-06, + "logits/chosen": -1.2128026485443115, + "logits/rejected": -1.0893349647521973, + "logps/chosen": -329.65692138671875, + "logps/rejected": -456.46087646484375, + "loss": 0.4662, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.4115194082260132, + "rewards/margins": 1.0705502033233643, + "rewards/rejected": -2.482069492340088, + "step": 7600 + }, + { + "epoch": 0.5, + "eval_logits/chosen": -1.2991713285446167, + "eval_logits/rejected": -1.1591788530349731, + "eval_logps/chosen": -382.1550598144531, + "eval_logps/rejected": -461.83013916015625, + "eval_loss": 0.5000800490379333, + "eval_rewards/accuracies": 0.6669999957084656, + "eval_rewards/chosen": -1.5015009641647339, + "eval_rewards/margins": 1.0006811618804932, + "eval_rewards/rejected": -2.5021822452545166, + "eval_runtime": 732.2161, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.366, + "step": 7600 + }, + { + "epoch": 0.5, + "learning_rate": 2.9522900048904534e-06, + "logits/chosen": -1.3992488384246826, + "logits/rejected": -1.1895803213119507, + "logps/chosen": -424.6064453125, + "logps/rejected": -467.15789794921875, + "loss": 0.5624, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8486826419830322, + "rewards/margins": 0.725730299949646, + "rewards/rejected": -2.5744128227233887, + "step": 7610 + }, + { + "epoch": 0.5, + "learning_rate": 2.9466731415704343e-06, + "logits/chosen": -1.306740164756775, + "logits/rejected": -1.1790627241134644, + "logps/chosen": -384.5773620605469, + "logps/rejected": -492.77264404296875, + "loss": 0.4353, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.6079070568084717, + "rewards/margins": 1.1256263256072998, + "rewards/rejected": -2.7335333824157715, + "step": 7620 + }, + { + "epoch": 0.5, + "learning_rate": 2.941053948184503e-06, + "logits/chosen": -1.3813174962997437, + "logits/rejected": -1.0688410997390747, + "logps/chosen": -429.5804748535156, + "logps/rejected": -484.577880859375, + "loss": 0.5808, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.4964145421981812, + "rewards/margins": 0.8990960121154785, + "rewards/rejected": -2.39551043510437, + "step": 7630 + }, + { + "epoch": 0.5, + "learning_rate": 2.935432454045125e-06, + "logits/chosen": -1.0862460136413574, + "logits/rejected": -1.1066715717315674, + "logps/chosen": -408.181640625, + "logps/rejected": -448.7203674316406, + "loss": 0.5967, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.7621729373931885, + "rewards/margins": 0.6174625158309937, + "rewards/rejected": -2.3796355724334717, + "step": 7640 + }, + { + "epoch": 0.5, + "learning_rate": 2.929808688476768e-06, + "logits/chosen": -1.3649405241012573, + "logits/rejected": -1.3180668354034424, + "logps/chosen": -405.2919921875, + "logps/rejected": -504.7835388183594, + "loss": 0.4175, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.6592957973480225, + "rewards/margins": 1.170940637588501, + "rewards/rejected": -2.8302364349365234, + "step": 7650 + }, + { + "epoch": 0.5, + "learning_rate": 2.924182680815748e-06, + "logits/chosen": -1.2287496328353882, + "logits/rejected": -1.1857244968414307, + "logps/chosen": -394.7705383300781, + "logps/rejected": -527.1818237304688, + "loss": 0.3751, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.6013991832733154, + "rewards/margins": 1.551189661026001, + "rewards/rejected": -3.1525888442993164, + "step": 7660 + }, + { + "epoch": 0.5, + "learning_rate": 2.9185544604100765e-06, + "logits/chosen": -0.9437403678894043, + "logits/rejected": -0.8567582368850708, + "logps/chosen": -395.3851013183594, + "logps/rejected": -465.4707946777344, + "loss": 0.6494, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.9947502613067627, + "rewards/margins": 0.7434107661247253, + "rewards/rejected": -2.7381606101989746, + "step": 7670 + }, + { + "epoch": 0.5, + "learning_rate": 2.9129240566193083e-06, + "logits/chosen": -1.505251169204712, + "logits/rejected": -1.1629559993743896, + "logps/chosen": -369.99041748046875, + "logps/rejected": -450.4754333496094, + "loss": 0.5025, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6791009902954102, + "rewards/margins": 0.9377298355102539, + "rewards/rejected": -2.616830825805664, + "step": 7680 + }, + { + "epoch": 0.5, + "learning_rate": 2.9072914988143874e-06, + "logits/chosen": -1.2632492780685425, + "logits/rejected": -1.0924867391586304, + "logps/chosen": -349.3762512207031, + "logps/rejected": -469.223388671875, + "loss": 0.3885, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.4794648885726929, + "rewards/margins": 1.2831852436065674, + "rewards/rejected": -2.7626500129699707, + "step": 7690 + }, + { + "epoch": 0.5, + "learning_rate": 2.9016568163774956e-06, + "logits/chosen": -1.3373229503631592, + "logits/rejected": -1.1682683229446411, + "logps/chosen": -305.866455078125, + "logps/rejected": -353.1499938964844, + "loss": 0.5059, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.336371660232544, + "rewards/margins": 0.8133636713027954, + "rewards/rejected": -2.149735450744629, + "step": 7700 + }, + { + "epoch": 0.5, + "eval_logits/chosen": -1.324633002281189, + "eval_logits/rejected": -1.1849044561386108, + "eval_logps/chosen": -373.3851318359375, + "eval_logps/rejected": -449.12884521484375, + "eval_loss": 0.4978525936603546, + "eval_rewards/accuracies": 0.6710000038146973, + "eval_rewards/chosen": -1.4138015508651733, + "eval_rewards/margins": 0.961367666721344, + "eval_rewards/rejected": -2.375169515609741, + "eval_runtime": 735.0378, + "eval_samples_per_second": 2.721, + "eval_steps_per_second": 1.36, + "step": 7700 + }, + { + "epoch": 0.5, + "learning_rate": 2.8960200387018942e-06, + "logits/chosen": -1.3530820608139038, + "logits/rejected": -1.199158787727356, + "logps/chosen": -462.70745849609375, + "logps/rejected": -492.1473693847656, + "loss": 0.5305, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.5497446060180664, + "rewards/margins": 0.7827426195144653, + "rewards/rejected": -2.332487106323242, + "step": 7710 + }, + { + "epoch": 0.51, + "learning_rate": 2.8903811951917792e-06, + "logits/chosen": -1.3843700885772705, + "logits/rejected": -1.33487868309021, + "logps/chosen": -311.75762939453125, + "logps/rejected": -350.55328369140625, + "loss": 0.4853, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.1347800493240356, + "rewards/margins": 0.859378457069397, + "rewards/rejected": -1.994158387184143, + "step": 7720 + }, + { + "epoch": 0.51, + "learning_rate": 2.88474031526212e-06, + "logits/chosen": -1.4788033962249756, + "logits/rejected": -1.3710378408432007, + "logps/chosen": -347.46319580078125, + "logps/rejected": -442.15655517578125, + "loss": 0.4774, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.4462344646453857, + "rewards/margins": 0.8126481175422668, + "rewards/rejected": -2.258882522583008, + "step": 7730 + }, + { + "epoch": 0.51, + "learning_rate": 2.879097428338509e-06, + "logits/chosen": -1.3819596767425537, + "logits/rejected": -1.0944236516952515, + "logps/chosen": -347.5988464355469, + "logps/rejected": -415.97509765625, + "loss": 0.5487, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3170385360717773, + "rewards/margins": 0.9070457220077515, + "rewards/rejected": -2.22408390045166, + "step": 7740 + }, + { + "epoch": 0.51, + "learning_rate": 2.8734525638570094e-06, + "logits/chosen": -1.379392147064209, + "logits/rejected": -1.3306528329849243, + "logps/chosen": -370.0411682128906, + "logps/rejected": -447.42645263671875, + "loss": 0.5758, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.3584692478179932, + "rewards/margins": 0.8975862264633179, + "rewards/rejected": -2.2560553550720215, + "step": 7750 + }, + { + "epoch": 0.51, + "learning_rate": 2.8678057512639982e-06, + "logits/chosen": -1.3406450748443604, + "logits/rejected": -1.2567927837371826, + "logps/chosen": -405.84051513671875, + "logps/rejected": -522.5167236328125, + "loss": 0.3867, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -1.1977407932281494, + "rewards/margins": 1.3873558044433594, + "rewards/rejected": -2.585096836090088, + "step": 7760 + }, + { + "epoch": 0.51, + "learning_rate": 2.8621570200160172e-06, + "logits/chosen": -0.9038798213005066, + "logits/rejected": -0.8531060218811035, + "logps/chosen": -302.6310729980469, + "logps/rejected": -423.69244384765625, + "loss": 0.4657, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.3406198024749756, + "rewards/margins": 1.2923932075500488, + "rewards/rejected": -2.6330132484436035, + "step": 7770 + }, + { + "epoch": 0.51, + "learning_rate": 2.856506399579615e-06, + "logits/chosen": -1.3094675540924072, + "logits/rejected": -1.25465989112854, + "logps/chosen": -426.06146240234375, + "logps/rejected": -501.49700927734375, + "loss": 0.5147, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.0769801139831543, + "rewards/margins": 0.9492477178573608, + "rewards/rejected": -3.0262279510498047, + "step": 7780 + }, + { + "epoch": 0.51, + "learning_rate": 2.8508539194311964e-06, + "logits/chosen": -1.1623289585113525, + "logits/rejected": -1.2776973247528076, + "logps/chosen": -444.77008056640625, + "logps/rejected": -559.510986328125, + "loss": 0.4635, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9103660583496094, + "rewards/margins": 1.0191086530685425, + "rewards/rejected": -2.929474353790283, + "step": 7790 + }, + { + "epoch": 0.51, + "learning_rate": 2.8451996090568656e-06, + "logits/chosen": -1.0837061405181885, + "logits/rejected": -0.8834303617477417, + "logps/chosen": -380.545654296875, + "logps/rejected": -475.828369140625, + "loss": 0.4464, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9440603256225586, + "rewards/margins": 1.1324434280395508, + "rewards/rejected": -3.0765042304992676, + "step": 7800 + }, + { + "epoch": 0.51, + "eval_logits/chosen": -1.0977903604507446, + "eval_logits/rejected": -0.9724917411804199, + "eval_logps/chosen": -452.9457702636719, + "eval_logps/rejected": -531.2132568359375, + "eval_loss": 0.5016809701919556, + "eval_rewards/accuracies": 0.6740000247955322, + "eval_rewards/chosen": -2.209408760070801, + "eval_rewards/margins": 0.9866045117378235, + "eval_rewards/rejected": -3.1960132122039795, + "eval_runtime": 732.3502, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.365, + "step": 7800 + }, + { + "epoch": 0.51, + "learning_rate": 2.839543497952276e-06, + "logits/chosen": -1.030879259109497, + "logits/rejected": -1.0171698331832886, + "logps/chosen": -380.8997497558594, + "logps/rejected": -486.53656005859375, + "loss": 0.4362, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9628963470458984, + "rewards/margins": 1.1434067487716675, + "rewards/rejected": -3.1063029766082764, + "step": 7810 + }, + { + "epoch": 0.51, + "learning_rate": 2.833885615622474e-06, + "logits/chosen": -0.986414909362793, + "logits/rejected": -0.8642165064811707, + "logps/chosen": -474.72515869140625, + "logps/rejected": -554.31201171875, + "loss": 0.6366, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.7214932441711426, + "rewards/margins": 0.7011139392852783, + "rewards/rejected": -3.422607898712158, + "step": 7820 + }, + { + "epoch": 0.51, + "learning_rate": 2.8282259915817454e-06, + "logits/chosen": -0.789636492729187, + "logits/rejected": -0.7618891596794128, + "logps/chosen": -339.92449951171875, + "logps/rejected": -497.02923583984375, + "loss": 0.4635, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9897979497909546, + "rewards/margins": 1.168931484222412, + "rewards/rejected": -3.1587295532226562, + "step": 7830 + }, + { + "epoch": 0.51, + "learning_rate": 2.8225646553534614e-06, + "logits/chosen": -0.812696099281311, + "logits/rejected": -0.7494589686393738, + "logps/chosen": -385.42608642578125, + "logps/rejected": -450.68621826171875, + "loss": 0.5923, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.8611011505126953, + "rewards/margins": 0.6855142712593079, + "rewards/rejected": -2.5466151237487793, + "step": 7840 + }, + { + "epoch": 0.51, + "learning_rate": 2.8169016364699255e-06, + "logits/chosen": -1.135119080543518, + "logits/rejected": -0.9809944033622742, + "logps/chosen": -436.8231506347656, + "logps/rejected": -525.0554809570312, + "loss": 0.5057, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.249351978302002, + "rewards/margins": 0.864822268486023, + "rewards/rejected": -3.1141743659973145, + "step": 7850 + }, + { + "epoch": 0.51, + "learning_rate": 2.811236964472217e-06, + "logits/chosen": -1.2407481670379639, + "logits/rejected": -1.075469732284546, + "logps/chosen": -495.60333251953125, + "logps/rejected": -545.838134765625, + "loss": 0.4944, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.8519678115844727, + "rewards/margins": 1.103547215461731, + "rewards/rejected": -2.955515146255493, + "step": 7860 + }, + { + "epoch": 0.51, + "learning_rate": 2.805570668910041e-06, + "logits/chosen": -0.8709837198257446, + "logits/rejected": -0.9303278923034668, + "logps/chosen": -379.8825988769531, + "logps/rejected": -545.008544921875, + "loss": 0.4578, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.1024327278137207, + "rewards/margins": 1.0162558555603027, + "rewards/rejected": -3.1186885833740234, + "step": 7870 + }, + { + "epoch": 0.52, + "learning_rate": 2.7999027793415695e-06, + "logits/chosen": -1.2885119915008545, + "logits/rejected": -0.8333857655525208, + "logps/chosen": -422.70928955078125, + "logps/rejected": -477.76019287109375, + "loss": 0.5059, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7779327630996704, + "rewards/margins": 0.9997657537460327, + "rewards/rejected": -2.7776987552642822, + "step": 7880 + }, + { + "epoch": 0.52, + "learning_rate": 2.794233325333293e-06, + "logits/chosen": -1.0390526056289673, + "logits/rejected": -0.8826634287834167, + "logps/chosen": -427.525634765625, + "logps/rejected": -531.1912231445312, + "loss": 0.4356, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.6924421787261963, + "rewards/margins": 1.273428201675415, + "rewards/rejected": -2.9658703804016113, + "step": 7890 + }, + { + "epoch": 0.52, + "learning_rate": 2.7885623364598597e-06, + "logits/chosen": -1.298937201499939, + "logits/rejected": -0.9922115206718445, + "logps/chosen": -453.6202697753906, + "logps/rejected": -565.6160888671875, + "loss": 0.3597, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.8992764949798584, + "rewards/margins": 1.5391474962234497, + "rewards/rejected": -3.4384243488311768, + "step": 7900 + }, + { + "epoch": 0.52, + "eval_logits/chosen": -1.0761736631393433, + "eval_logits/rejected": -0.9468325972557068, + "eval_logps/chosen": -403.9175720214844, + "eval_logps/rejected": -494.2936706542969, + "eval_loss": 0.495647668838501, + "eval_rewards/accuracies": 0.6725000143051147, + "eval_rewards/chosen": -1.7191258668899536, + "eval_rewards/margins": 1.1076916456222534, + "eval_rewards/rejected": -2.826817274093628, + "eval_runtime": 733.0812, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 7900 + }, + { + "epoch": 0.52, + "learning_rate": 2.782889842303926e-06, + "logits/chosen": -1.0714633464813232, + "logits/rejected": -1.0015760660171509, + "logps/chosen": -360.4320983886719, + "logps/rejected": -431.151123046875, + "loss": 0.5618, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0020298957824707, + "rewards/margins": 0.7621687650680542, + "rewards/rejected": -2.7641987800598145, + "step": 7910 + }, + { + "epoch": 0.52, + "learning_rate": 2.7772158724559987e-06, + "logits/chosen": -1.0210740566253662, + "logits/rejected": -0.7624063491821289, + "logps/chosen": -365.82025146484375, + "logps/rejected": -589.4786376953125, + "loss": 0.36, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.4871119260787964, + "rewards/margins": 1.861876130104065, + "rewards/rejected": -3.3489882946014404, + "step": 7920 + }, + { + "epoch": 0.52, + "learning_rate": 2.7715404565142856e-06, + "logits/chosen": -0.7972566485404968, + "logits/rejected": -0.9107509851455688, + "logps/chosen": -411.510009765625, + "logps/rejected": -491.3306579589844, + "loss": 0.5595, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.1154580116271973, + "rewards/margins": 0.8604329228401184, + "rewards/rejected": -2.975890636444092, + "step": 7930 + }, + { + "epoch": 0.52, + "learning_rate": 2.7658636240845354e-06, + "logits/chosen": -1.2706204652786255, + "logits/rejected": -1.1921221017837524, + "logps/chosen": -449.60516357421875, + "logps/rejected": -611.0333251953125, + "loss": 0.3961, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.261568307876587, + "rewards/margins": 1.468100905418396, + "rewards/rejected": -3.7296690940856934, + "step": 7940 + }, + { + "epoch": 0.52, + "learning_rate": 2.7601854047798872e-06, + "logits/chosen": -0.7478612661361694, + "logits/rejected": -0.800074577331543, + "logps/chosen": -465.3731384277344, + "logps/rejected": -553.3946533203125, + "loss": 0.6341, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.4144978523254395, + "rewards/margins": 0.6860623359680176, + "rewards/rejected": -3.100559949874878, + "step": 7950 + }, + { + "epoch": 0.52, + "learning_rate": 2.7545058282207148e-06, + "logits/chosen": -0.6742497682571411, + "logits/rejected": -0.6114516258239746, + "logps/chosen": -422.21807861328125, + "logps/rejected": -478.6226501464844, + "loss": 0.5483, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.106762647628784, + "rewards/margins": 0.8698352575302124, + "rewards/rejected": -2.976598024368286, + "step": 7960 + }, + { + "epoch": 0.52, + "learning_rate": 2.748824924034471e-06, + "logits/chosen": -0.9385892152786255, + "logits/rejected": -0.8127745389938354, + "logps/chosen": -480.69482421875, + "logps/rejected": -532.44775390625, + "loss": 0.6853, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.6128458976745605, + "rewards/margins": 0.7064965963363647, + "rewards/rejected": -3.319342851638794, + "step": 7970 + }, + { + "epoch": 0.52, + "learning_rate": 2.743142721855536e-06, + "logits/chosen": -0.4976627230644226, + "logits/rejected": -0.6767405271530151, + "logps/chosen": -345.3768310546875, + "logps/rejected": -402.8832702636719, + "loss": 0.5117, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9351670742034912, + "rewards/margins": 0.6585490107536316, + "rewards/rejected": -2.5937161445617676, + "step": 7980 + }, + { + "epoch": 0.52, + "learning_rate": 2.737459251325058e-06, + "logits/chosen": -0.8807660937309265, + "logits/rejected": -0.729613184928894, + "logps/chosen": -452.9046325683594, + "logps/rejected": -510.9497985839844, + "loss": 0.4951, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.829310655593872, + "rewards/margins": 0.8057653307914734, + "rewards/rejected": -2.6350760459899902, + "step": 7990 + }, + { + "epoch": 0.52, + "learning_rate": 2.731774542090804e-06, + "logits/chosen": -0.5301181674003601, + "logits/rejected": -0.5973738431930542, + "logps/chosen": -405.48492431640625, + "logps/rejected": -436.1067810058594, + "loss": 0.6685, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.127089262008667, + "rewards/margins": 0.4965682625770569, + "rewards/rejected": -2.623657703399658, + "step": 8000 + }, + { + "epoch": 0.52, + "eval_logits/chosen": -0.8313817977905273, + "eval_logits/rejected": -0.7170847058296204, + "eval_logps/chosen": -446.3575134277344, + "eval_logps/rejected": -524.3576049804688, + "eval_loss": 0.4940214455127716, + "eval_rewards/accuracies": 0.6694999933242798, + "eval_rewards/chosen": -2.1435256004333496, + "eval_rewards/margins": 0.9839317798614502, + "eval_rewards/rejected": -3.127457618713379, + "eval_runtime": 733.0072, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 8000 + }, + { + "epoch": 0.52, + "learning_rate": 2.7260886238070034e-06, + "logits/chosen": -0.9801632761955261, + "logits/rejected": -0.8519521951675415, + "logps/chosen": -426.9603576660156, + "logps/rejected": -517.7711181640625, + "loss": 0.4641, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3049042224884033, + "rewards/margins": 0.9551888704299927, + "rewards/rejected": -3.2600929737091064, + "step": 8010 + }, + { + "epoch": 0.52, + "learning_rate": 2.72040152613419e-06, + "logits/chosen": -0.8683644533157349, + "logits/rejected": -0.7820503115653992, + "logps/chosen": -433.59735107421875, + "logps/rejected": -472.0309143066406, + "loss": 0.4193, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.167222261428833, + "rewards/margins": 1.2165911197662354, + "rewards/rejected": -3.3838133811950684, + "step": 8020 + }, + { + "epoch": 0.53, + "learning_rate": 2.7147132787390516e-06, + "logits/chosen": -1.0270106792449951, + "logits/rejected": -0.7757326364517212, + "logps/chosen": -414.59613037109375, + "logps/rejected": -508.03814697265625, + "loss": 0.4447, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8711802959442139, + "rewards/margins": 1.1042753458023071, + "rewards/rejected": -2.9754557609558105, + "step": 8030 + }, + { + "epoch": 0.53, + "learning_rate": 2.709023911294273e-06, + "logits/chosen": -1.136345624923706, + "logits/rejected": -0.9073754549026489, + "logps/chosen": -395.5877990722656, + "logps/rejected": -545.9440307617188, + "loss": 0.4058, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.5326893329620361, + "rewards/margins": 1.8125578165054321, + "rewards/rejected": -3.3452467918395996, + "step": 8040 + }, + { + "epoch": 0.53, + "learning_rate": 2.7033334534783806e-06, + "logits/chosen": -1.1330026388168335, + "logits/rejected": -1.2662372589111328, + "logps/chosen": -347.5755920410156, + "logps/rejected": -466.73443603515625, + "loss": 0.5626, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.4842970371246338, + "rewards/margins": 1.0295974016189575, + "rewards/rejected": -2.5138943195343018, + "step": 8050 + }, + { + "epoch": 0.53, + "learning_rate": 2.697641934975592e-06, + "logits/chosen": -1.2012157440185547, + "logits/rejected": -1.0244762897491455, + "logps/chosen": -388.85107421875, + "logps/rejected": -484.35498046875, + "loss": 0.5007, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.6106462478637695, + "rewards/margins": 1.280577540397644, + "rewards/rejected": -2.891223907470703, + "step": 8060 + }, + { + "epoch": 0.53, + "learning_rate": 2.691949385475654e-06, + "logits/chosen": -1.198803186416626, + "logits/rejected": -1.0046731233596802, + "logps/chosen": -436.41021728515625, + "logps/rejected": -520.1088256835938, + "loss": 0.5023, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9251935482025146, + "rewards/margins": 1.0926984548568726, + "rewards/rejected": -3.0178921222686768, + "step": 8070 + }, + { + "epoch": 0.53, + "learning_rate": 2.6862558346736937e-06, + "logits/chosen": -1.1371890306472778, + "logits/rejected": -0.9129287600517273, + "logps/chosen": -407.1488037109375, + "logps/rejected": -593.5284423828125, + "loss": 0.3355, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6879653930664062, + "rewards/margins": 1.915813684463501, + "rewards/rejected": -3.603778839111328, + "step": 8080 + }, + { + "epoch": 0.53, + "learning_rate": 2.6805613122700617e-06, + "logits/chosen": -1.0117746591567993, + "logits/rejected": -0.9848273992538452, + "logps/chosen": -440.2730407714844, + "logps/rejected": -555.3220825195312, + "loss": 0.5118, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.1670055389404297, + "rewards/margins": 1.1410691738128662, + "rewards/rejected": -3.308074474334717, + "step": 8090 + }, + { + "epoch": 0.53, + "learning_rate": 2.674865847970176e-06, + "logits/chosen": -1.0662400722503662, + "logits/rejected": -0.8855364918708801, + "logps/chosen": -409.4464416503906, + "logps/rejected": -533.459228515625, + "loss": 0.5494, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.0183000564575195, + "rewards/margins": 1.015644907951355, + "rewards/rejected": -3.033945322036743, + "step": 8100 + }, + { + "epoch": 0.53, + "eval_logits/chosen": -1.0209530591964722, + "eval_logits/rejected": -0.8909984827041626, + "eval_logps/chosen": -445.6345520019531, + "eval_logps/rejected": -532.8621826171875, + "eval_loss": 0.49142247438430786, + "eval_rewards/accuracies": 0.6654999852180481, + "eval_rewards/chosen": -2.136295795440674, + "eval_rewards/margins": 1.076207160949707, + "eval_rewards/rejected": -3.212502956390381, + "eval_runtime": 731.4903, + "eval_samples_per_second": 2.734, + "eval_steps_per_second": 1.367, + "step": 8100 + }, + { + "epoch": 0.53, + "learning_rate": 2.669169471484368e-06, + "logits/chosen": -0.7829369306564331, + "logits/rejected": -0.8367104530334473, + "logps/chosen": -400.9513244628906, + "logps/rejected": -442.726806640625, + "loss": 0.6904, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.3589324951171875, + "rewards/margins": 0.44415682554244995, + "rewards/rejected": -2.8030896186828613, + "step": 8110 + }, + { + "epoch": 0.53, + "learning_rate": 2.6634722125277278e-06, + "logits/chosen": -1.1393558979034424, + "logits/rejected": -0.8891308903694153, + "logps/chosen": -481.6310119628906, + "logps/rejected": -586.5345458984375, + "loss": 0.5243, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.4951107501983643, + "rewards/margins": 0.9670877456665039, + "rewards/rejected": -3.4621987342834473, + "step": 8120 + }, + { + "epoch": 0.53, + "learning_rate": 2.6577741008199498e-06, + "logits/chosen": -0.939818263053894, + "logits/rejected": -0.8135367631912231, + "logps/chosen": -490.2794494628906, + "logps/rejected": -577.6835327148438, + "loss": 0.409, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.341212511062622, + "rewards/margins": 1.474454402923584, + "rewards/rejected": -3.815666913986206, + "step": 8130 + }, + { + "epoch": 0.53, + "learning_rate": 2.652075166085175e-06, + "logits/chosen": -0.8981655240058899, + "logits/rejected": -0.9567068815231323, + "logps/chosen": -460.9468688964844, + "logps/rejected": -618.2373046875, + "loss": 0.491, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.2636752128601074, + "rewards/margins": 1.3374521732330322, + "rewards/rejected": -3.6011271476745605, + "step": 8140 + }, + { + "epoch": 0.53, + "learning_rate": 2.6463754380518395e-06, + "logits/chosen": -0.8287490010261536, + "logits/rejected": -0.7122872471809387, + "logps/chosen": -475.3302307128906, + "logps/rejected": -512.5914306640625, + "loss": 0.5309, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.443725347518921, + "rewards/margins": 0.9208828806877136, + "rewards/rejected": -3.3646087646484375, + "step": 8150 + }, + { + "epoch": 0.53, + "learning_rate": 2.6406749464525167e-06, + "logits/chosen": -1.278389811515808, + "logits/rejected": -0.9623058438301086, + "logps/chosen": -380.12249755859375, + "logps/rejected": -452.919921875, + "loss": 0.4098, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.4656339883804321, + "rewards/margins": 1.200292706489563, + "rewards/rejected": -2.665926694869995, + "step": 8160 + }, + { + "epoch": 0.53, + "learning_rate": 2.634973721023762e-06, + "logits/chosen": -1.3094470500946045, + "logits/rejected": -1.1664118766784668, + "logps/chosen": -468.9601135253906, + "logps/rejected": -489.9962463378906, + "loss": 0.5648, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1449787616729736, + "rewards/margins": 0.7047473192214966, + "rewards/rejected": -2.849726438522339, + "step": 8170 + }, + { + "epoch": 0.54, + "learning_rate": 2.6292717915059605e-06, + "logits/chosen": -1.3345282077789307, + "logits/rejected": -1.1698658466339111, + "logps/chosen": -482.92279052734375, + "logps/rejected": -533.3839111328125, + "loss": 0.4298, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0729269981384277, + "rewards/margins": 1.0665416717529297, + "rewards/rejected": -3.1394686698913574, + "step": 8180 + }, + { + "epoch": 0.54, + "learning_rate": 2.6235691876431706e-06, + "logits/chosen": -1.2751821279525757, + "logits/rejected": -1.2013987302780151, + "logps/chosen": -439.2770080566406, + "logps/rejected": -522.8220825195312, + "loss": 0.5507, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.2066140174865723, + "rewards/margins": 0.7794591784477234, + "rewards/rejected": -2.9860730171203613, + "step": 8190 + }, + { + "epoch": 0.54, + "learning_rate": 2.6178659391829673e-06, + "logits/chosen": -1.326781988143921, + "logits/rejected": -1.0384869575500488, + "logps/chosen": -414.6600646972656, + "logps/rejected": -479.1424255371094, + "loss": 0.4703, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7718207836151123, + "rewards/margins": 1.0532562732696533, + "rewards/rejected": -2.8250768184661865, + "step": 8200 + }, + { + "epoch": 0.54, + "eval_logits/chosen": -1.1886258125305176, + "eval_logits/rejected": -1.055037498474121, + "eval_logps/chosen": -433.6510314941406, + "eval_logps/rejected": -508.3775634765625, + "eval_loss": 0.49489960074424744, + "eval_rewards/accuracies": 0.6660000085830688, + "eval_rewards/chosen": -2.016460657119751, + "eval_rewards/margins": 0.9511955380439758, + "eval_rewards/rejected": -2.9676566123962402, + "eval_runtime": 734.8366, + "eval_samples_per_second": 2.722, + "eval_steps_per_second": 1.361, + "step": 8200 + }, + { + "epoch": 0.54, + "learning_rate": 2.6121620758762877e-06, + "logits/chosen": -1.2698452472686768, + "logits/rejected": -0.9875261187553406, + "logps/chosen": -390.28704833984375, + "logps/rejected": -475.6351013183594, + "loss": 0.4836, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9820274114608765, + "rewards/margins": 0.8614705801010132, + "rewards/rejected": -2.8434982299804688, + "step": 8210 + }, + { + "epoch": 0.54, + "learning_rate": 2.606457627477277e-06, + "logits/chosen": -1.0521068572998047, + "logits/rejected": -0.9721614122390747, + "logps/chosen": -345.18341064453125, + "logps/rejected": -463.9532165527344, + "loss": 0.4758, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.6955410242080688, + "rewards/margins": 1.1541770696640015, + "rewards/rejected": -2.8497185707092285, + "step": 8220 + }, + { + "epoch": 0.54, + "learning_rate": 2.6007526237431324e-06, + "logits/chosen": -1.3272507190704346, + "logits/rejected": -1.214674472808838, + "logps/chosen": -360.69989013671875, + "logps/rejected": -493.56854248046875, + "loss": 0.4782, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7867038249969482, + "rewards/margins": 1.1875064373016357, + "rewards/rejected": -2.974210023880005, + "step": 8230 + }, + { + "epoch": 0.54, + "learning_rate": 2.5950470944339478e-06, + "logits/chosen": -1.1821718215942383, + "logits/rejected": -1.211446762084961, + "logps/chosen": -405.30413818359375, + "logps/rejected": -467.179931640625, + "loss": 0.5559, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.8350213766098022, + "rewards/margins": 0.6537161469459534, + "rewards/rejected": -2.4887375831604004, + "step": 8240 + }, + { + "epoch": 0.54, + "learning_rate": 2.58934106931256e-06, + "logits/chosen": -1.1724306344985962, + "logits/rejected": -1.018654704093933, + "logps/chosen": -437.35595703125, + "logps/rejected": -499.43572998046875, + "loss": 0.5731, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1658389568328857, + "rewards/margins": 0.8125737905502319, + "rewards/rejected": -2.978412628173828, + "step": 8250 + }, + { + "epoch": 0.54, + "learning_rate": 2.58363457814439e-06, + "logits/chosen": -1.2482324838638306, + "logits/rejected": -0.9831002354621887, + "logps/chosen": -450.75244140625, + "logps/rejected": -523.3827514648438, + "loss": 0.6366, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.4008052349090576, + "rewards/margins": 0.8635374903678894, + "rewards/rejected": -3.26434326171875, + "step": 8260 + }, + { + "epoch": 0.54, + "learning_rate": 2.5779276506972924e-06, + "logits/chosen": -1.1220371723175049, + "logits/rejected": -1.1320124864578247, + "logps/chosen": -439.0234375, + "logps/rejected": -486.7811584472656, + "loss": 0.554, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0707054138183594, + "rewards/margins": 0.8713995218276978, + "rewards/rejected": -2.9421050548553467, + "step": 8270 + }, + { + "epoch": 0.54, + "learning_rate": 2.5722203167413945e-06, + "logits/chosen": -1.2530734539031982, + "logits/rejected": -1.1008774042129517, + "logps/chosen": -472.61737060546875, + "logps/rejected": -483.79278564453125, + "loss": 0.495, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8931972980499268, + "rewards/margins": 0.9309617280960083, + "rewards/rejected": -2.8241589069366455, + "step": 8280 + }, + { + "epoch": 0.54, + "learning_rate": 2.5665126060489476e-06, + "logits/chosen": -1.3096177577972412, + "logits/rejected": -1.2289804220199585, + "logps/chosen": -399.1604919433594, + "logps/rejected": -502.7049255371094, + "loss": 0.5053, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1105847358703613, + "rewards/margins": 0.7469784021377563, + "rewards/rejected": -2.857563018798828, + "step": 8290 + }, + { + "epoch": 0.54, + "learning_rate": 2.560804548394165e-06, + "logits/chosen": -1.165805459022522, + "logits/rejected": -0.8079797625541687, + "logps/chosen": -437.4986877441406, + "logps/rejected": -492.98358154296875, + "loss": 0.4901, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.902182936668396, + "rewards/margins": 1.007571816444397, + "rewards/rejected": -2.909754991531372, + "step": 8300 + }, + { + "epoch": 0.54, + "eval_logits/chosen": -1.2041014432907104, + "eval_logits/rejected": -1.072398066520691, + "eval_logps/chosen": -416.77789306640625, + "eval_logps/rejected": -487.3052978515625, + "eval_loss": 0.4975956082344055, + "eval_rewards/accuracies": 0.6635000109672546, + "eval_rewards/chosen": -1.8477294445037842, + "eval_rewards/margins": 0.9092043042182922, + "eval_rewards/rejected": -2.75693416595459, + "eval_runtime": 731.7183, + "eval_samples_per_second": 2.733, + "eval_steps_per_second": 1.367, + "step": 8300 + }, + { + "epoch": 0.54, + "learning_rate": 2.5550961735530734e-06, + "logits/chosen": -0.9673964381217957, + "logits/rejected": -1.094525933265686, + "logps/chosen": -307.0442810058594, + "logps/rejected": -424.9918518066406, + "loss": 0.4957, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.4400845766067505, + "rewards/margins": 0.8735688924789429, + "rewards/rejected": -2.3136534690856934, + "step": 8310 + }, + { + "epoch": 0.54, + "learning_rate": 2.549387511303351e-06, + "logits/chosen": -1.1674532890319824, + "logits/rejected": -1.2690563201904297, + "logps/chosen": -357.15203857421875, + "logps/rejected": -461.42706298828125, + "loss": 0.5702, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.874151587486267, + "rewards/margins": 0.5999444127082825, + "rewards/rejected": -2.4740958213806152, + "step": 8320 + }, + { + "epoch": 0.55, + "learning_rate": 2.5436785914241774e-06, + "logits/chosen": -1.0933245420455933, + "logits/rejected": -1.0153937339782715, + "logps/chosen": -414.09619140625, + "logps/rejected": -490.25152587890625, + "loss": 0.4778, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.151369571685791, + "rewards/margins": 1.081249475479126, + "rewards/rejected": -3.232619524002075, + "step": 8330 + }, + { + "epoch": 0.55, + "learning_rate": 2.5379694436960746e-06, + "logits/chosen": -1.2745726108551025, + "logits/rejected": -1.200372338294983, + "logps/chosen": -446.69580078125, + "logps/rejected": -548.1271362304688, + "loss": 0.5295, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0068485736846924, + "rewards/margins": 0.9086639285087585, + "rewards/rejected": -2.9155125617980957, + "step": 8340 + }, + { + "epoch": 0.55, + "learning_rate": 2.5322600979007533e-06, + "logits/chosen": -1.3381099700927734, + "logits/rejected": -1.088611364364624, + "logps/chosen": -390.39862060546875, + "logps/rejected": -444.66998291015625, + "loss": 0.5258, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7790319919586182, + "rewards/margins": 0.7497051358222961, + "rewards/rejected": -2.5287370681762695, + "step": 8350 + }, + { + "epoch": 0.55, + "learning_rate": 2.5265505838209592e-06, + "logits/chosen": -1.216956615447998, + "logits/rejected": -1.059798240661621, + "logps/chosen": -504.1844177246094, + "logps/rejected": -533.2293701171875, + "loss": 0.6299, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.4722084999084473, + "rewards/margins": 0.7545827627182007, + "rewards/rejected": -3.2267906665802, + "step": 8360 + }, + { + "epoch": 0.55, + "learning_rate": 2.520840931240314e-06, + "logits/chosen": -1.2149550914764404, + "logits/rejected": -0.8863165974617004, + "logps/chosen": -431.2649841308594, + "logps/rejected": -461.96856689453125, + "loss": 0.4361, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.2145636081695557, + "rewards/margins": 0.9442449808120728, + "rewards/rejected": -3.158808469772339, + "step": 8370 + }, + { + "epoch": 0.55, + "learning_rate": 2.515131169943162e-06, + "logits/chosen": -0.7880765795707703, + "logits/rejected": -0.7995393872261047, + "logps/chosen": -510.5859375, + "logps/rejected": -632.1096801757812, + "loss": 0.4487, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.5249135494232178, + "rewards/margins": 1.2849611043930054, + "rewards/rejected": -3.8098747730255127, + "step": 8380 + }, + { + "epoch": 0.55, + "learning_rate": 2.509421329714416e-06, + "logits/chosen": -0.7812287211418152, + "logits/rejected": -0.8888276219367981, + "logps/chosen": -446.288818359375, + "logps/rejected": -559.374267578125, + "loss": 0.5248, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.401496171951294, + "rewards/margins": 0.918934166431427, + "rewards/rejected": -3.320430278778076, + "step": 8390 + }, + { + "epoch": 0.55, + "learning_rate": 2.5037114403393987e-06, + "logits/chosen": -0.9296455383300781, + "logits/rejected": -0.7097471952438354, + "logps/chosen": -420.11614990234375, + "logps/rejected": -490.839111328125, + "loss": 0.4759, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.091357707977295, + "rewards/margins": 1.0450347661972046, + "rewards/rejected": -3.13639235496521, + "step": 8400 + }, + { + "epoch": 0.55, + "eval_logits/chosen": -1.0123143196105957, + "eval_logits/rejected": -0.8860378861427307, + "eval_logps/chosen": -479.3096008300781, + "eval_logps/rejected": -566.3602905273438, + "eval_loss": 0.4948574900627136, + "eval_rewards/accuracies": 0.6654999852180481, + "eval_rewards/chosen": -2.47304630279541, + "eval_rewards/margins": 1.0744378566741943, + "eval_rewards/rejected": -3.5474843978881836, + "eval_runtime": 732.3944, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.365, + "step": 8400 + }, + { + "epoch": 0.55, + "learning_rate": 2.4980015316036908e-06, + "logits/chosen": -0.8672155141830444, + "logits/rejected": -0.8484716415405273, + "logps/chosen": -402.535888671875, + "logps/rejected": -579.1514892578125, + "loss": 0.404, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.2672619819641113, + "rewards/margins": 1.5502973794937134, + "rewards/rejected": -3.817559003829956, + "step": 8410 + }, + { + "epoch": 0.55, + "learning_rate": 2.4922916332929725e-06, + "logits/chosen": -1.1366407871246338, + "logits/rejected": -1.1110883951187134, + "logps/chosen": -463.5205078125, + "logps/rejected": -523.5811767578125, + "loss": 0.4977, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2809627056121826, + "rewards/margins": 0.9967510104179382, + "rewards/rejected": -3.2777132987976074, + "step": 8420 + }, + { + "epoch": 0.55, + "learning_rate": 2.4865817751928716e-06, + "logits/chosen": -1.0972706079483032, + "logits/rejected": -1.0200917720794678, + "logps/chosen": -428.05413818359375, + "logps/rejected": -609.2821044921875, + "loss": 0.3714, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.3256542682647705, + "rewards/margins": 1.548058271408081, + "rewards/rejected": -3.8737120628356934, + "step": 8430 + }, + { + "epoch": 0.55, + "learning_rate": 2.4808719870888037e-06, + "logits/chosen": -0.9892493486404419, + "logits/rejected": -0.7642117738723755, + "logps/chosen": -451.3265686035156, + "logps/rejected": -560.0855102539062, + "loss": 0.4537, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3365511894226074, + "rewards/margins": 1.420327067375183, + "rewards/rejected": -3.75687837600708, + "step": 8440 + }, + { + "epoch": 0.55, + "learning_rate": 2.4751622987658206e-06, + "logits/chosen": -1.2597066164016724, + "logits/rejected": -1.1139386892318726, + "logps/chosen": -487.68646240234375, + "logps/rejected": -580.7915649414062, + "loss": 0.5196, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.497361898422241, + "rewards/margins": 1.0384573936462402, + "rewards/rejected": -3.5358192920684814, + "step": 8450 + }, + { + "epoch": 0.55, + "learning_rate": 2.4694527400084546e-06, + "logits/chosen": -1.1042139530181885, + "logits/rejected": -0.9779818654060364, + "logps/chosen": -441.48944091796875, + "logps/rejected": -536.3157348632812, + "loss": 0.4901, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1924479007720947, + "rewards/margins": 1.0067086219787598, + "rewards/rejected": -3.1991565227508545, + "step": 8460 + }, + { + "epoch": 0.55, + "learning_rate": 2.4637433406005607e-06, + "logits/chosen": -1.1094534397125244, + "logits/rejected": -1.2497950792312622, + "logps/chosen": -571.42431640625, + "logps/rejected": -641.7213134765625, + "loss": 0.5045, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.6169791221618652, + "rewards/margins": 0.9834015965461731, + "rewards/rejected": -3.6003806591033936, + "step": 8470 + }, + { + "epoch": 0.55, + "learning_rate": 2.4580341303251628e-06, + "logits/chosen": -0.862544059753418, + "logits/rejected": -0.7224100828170776, + "logps/chosen": -508.24090576171875, + "logps/rejected": -622.977783203125, + "loss": 0.3792, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.4828972816467285, + "rewards/margins": 1.511356234550476, + "rewards/rejected": -3.994253635406494, + "step": 8480 + }, + { + "epoch": 0.56, + "learning_rate": 2.4523251389642984e-06, + "logits/chosen": -1.0363171100616455, + "logits/rejected": -0.7067098617553711, + "logps/chosen": -539.4168701171875, + "logps/rejected": -659.4783935546875, + "loss": 0.422, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.8485755920410156, + "rewards/margins": 1.571999430656433, + "rewards/rejected": -4.420575141906738, + "step": 8490 + }, + { + "epoch": 0.56, + "learning_rate": 2.4466163962988626e-06, + "logits/chosen": -1.2876355648040771, + "logits/rejected": -0.9966632723808289, + "logps/chosen": -529.1260375976562, + "logps/rejected": -556.1248779296875, + "loss": 0.5511, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.485250949859619, + "rewards/margins": 1.257182002067566, + "rewards/rejected": -3.7424328327178955, + "step": 8500 + }, + { + "epoch": 0.56, + "eval_logits/chosen": -0.9928444623947144, + "eval_logits/rejected": -0.8652651906013489, + "eval_logps/chosen": -498.1315612792969, + "eval_logps/rejected": -596.16943359375, + "eval_loss": 0.49668240547180176, + "eval_rewards/accuracies": 0.6690000295639038, + "eval_rewards/chosen": -2.6612660884857178, + "eval_rewards/margins": 1.1843092441558838, + "eval_rewards/rejected": -3.8455750942230225, + "eval_runtime": 731.4494, + "eval_samples_per_second": 2.734, + "eval_steps_per_second": 1.367, + "step": 8500 + }, + { + "epoch": 0.56, + "learning_rate": 2.4409079321084543e-06, + "logits/chosen": -1.0816056728363037, + "logits/rejected": -1.2254623174667358, + "logps/chosen": -443.826416015625, + "logps/rejected": -604.9515380859375, + "loss": 0.4718, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.287067413330078, + "rewards/margins": 1.3118616342544556, + "rewards/rejected": -3.598928928375244, + "step": 8510 + }, + { + "epoch": 0.56, + "learning_rate": 2.4351997761712184e-06, + "logits/chosen": -1.3557398319244385, + "logits/rejected": -0.8279944658279419, + "logps/chosen": -487.305419921875, + "logps/rejected": -549.6810913085938, + "loss": 0.405, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.419170618057251, + "rewards/margins": 1.2908157110214233, + "rewards/rejected": -3.7099862098693848, + "step": 8520 + }, + { + "epoch": 0.56, + "learning_rate": 2.4294919582636933e-06, + "logits/chosen": -1.2039768695831299, + "logits/rejected": -0.9999167323112488, + "logps/chosen": -412.7548828125, + "logps/rejected": -518.6639404296875, + "loss": 0.4139, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.016523838043213, + "rewards/margins": 1.177484154701233, + "rewards/rejected": -3.1940078735351562, + "step": 8530 + }, + { + "epoch": 0.56, + "learning_rate": 2.423784508160652e-06, + "logits/chosen": -1.1773066520690918, + "logits/rejected": -1.0269687175750732, + "logps/chosen": -514.6947631835938, + "logps/rejected": -588.3440551757812, + "loss": 0.4267, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.5946402549743652, + "rewards/margins": 1.2148663997650146, + "rewards/rejected": -3.8095061779022217, + "step": 8540 + }, + { + "epoch": 0.56, + "learning_rate": 2.418077455634951e-06, + "logits/chosen": -1.0692018270492554, + "logits/rejected": -1.0105565786361694, + "logps/chosen": -440.36785888671875, + "logps/rejected": -564.6529541015625, + "loss": 0.5634, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.2379679679870605, + "rewards/margins": 0.967279314994812, + "rewards/rejected": -3.205247163772583, + "step": 8550 + }, + { + "epoch": 0.56, + "learning_rate": 2.4123708304573714e-06, + "logits/chosen": -1.2759517431259155, + "logits/rejected": -1.0008412599563599, + "logps/chosen": -499.835693359375, + "logps/rejected": -604.622314453125, + "loss": 0.5107, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.1006827354431152, + "rewards/margins": 1.21485435962677, + "rewards/rejected": -3.3155369758605957, + "step": 8560 + }, + { + "epoch": 0.56, + "learning_rate": 2.406664662396465e-06, + "logits/chosen": -0.8560956120491028, + "logits/rejected": -0.8265968561172485, + "logps/chosen": -431.65948486328125, + "logps/rejected": -494.79718017578125, + "loss": 0.619, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.472776174545288, + "rewards/margins": 0.7881981730461121, + "rewards/rejected": -3.260974407196045, + "step": 8570 + }, + { + "epoch": 0.56, + "learning_rate": 2.4009589812184012e-06, + "logits/chosen": -1.178475260734558, + "logits/rejected": -0.8446548581123352, + "logps/chosen": -381.1634216308594, + "logps/rejected": -413.203857421875, + "loss": 0.4929, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.779435157775879, + "rewards/margins": 0.8515142202377319, + "rewards/rejected": -2.6309492588043213, + "step": 8580 + }, + { + "epoch": 0.56, + "learning_rate": 2.3952538166868073e-06, + "logits/chosen": -1.0692102909088135, + "logits/rejected": -1.1181179285049438, + "logps/chosen": -401.1435546875, + "logps/rejected": -496.79901123046875, + "loss": 0.4597, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8359718322753906, + "rewards/margins": 1.1224865913391113, + "rewards/rejected": -2.958458423614502, + "step": 8590 + }, + { + "epoch": 0.56, + "learning_rate": 2.389549198562616e-06, + "logits/chosen": -1.2879879474639893, + "logits/rejected": -0.9037203788757324, + "logps/chosen": -403.9749450683594, + "logps/rejected": -526.24658203125, + "loss": 0.4126, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.7789331674575806, + "rewards/margins": 1.5225181579589844, + "rewards/rejected": -3.3014514446258545, + "step": 8600 + }, + { + "epoch": 0.56, + "eval_logits/chosen": -1.238702654838562, + "eval_logits/rejected": -1.1029170751571655, + "eval_logps/chosen": -414.6830749511719, + "eval_logps/rejected": -496.90240478515625, + "eval_loss": 0.4945288598537445, + "eval_rewards/accuracies": 0.6664999723434448, + "eval_rewards/chosen": -1.8267810344696045, + "eval_rewards/margins": 1.0261238813400269, + "eval_rewards/rejected": -2.852905035018921, + "eval_runtime": 732.9791, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.364, + "step": 8600 + }, + { + "epoch": 0.56, + "learning_rate": 2.3838451566039098e-06, + "logits/chosen": -1.4302728176116943, + "logits/rejected": -1.2620681524276733, + "logps/chosen": -432.6902770996094, + "logps/rejected": -485.573974609375, + "loss": 0.5606, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.941802978515625, + "rewards/margins": 0.630177915096283, + "rewards/rejected": -2.5719809532165527, + "step": 8610 + }, + { + "epoch": 0.56, + "learning_rate": 2.3781417205657662e-06, + "logits/chosen": -1.325465440750122, + "logits/rejected": -1.1412758827209473, + "logps/chosen": -370.631103515625, + "logps/rejected": -435.3427734375, + "loss": 0.5132, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.7287461757659912, + "rewards/margins": 1.029188871383667, + "rewards/rejected": -2.757934808731079, + "step": 8620 + }, + { + "epoch": 0.56, + "learning_rate": 2.3724389202001006e-06, + "logits/chosen": -1.1985505819320679, + "logits/rejected": -0.9390853643417358, + "logps/chosen": -383.4583435058594, + "logps/rejected": -473.318603515625, + "loss": 0.4625, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8113489151000977, + "rewards/margins": 1.1155695915222168, + "rewards/rejected": -2.9269185066223145, + "step": 8630 + }, + { + "epoch": 0.57, + "learning_rate": 2.366736785255514e-06, + "logits/chosen": -1.199562907218933, + "logits/rejected": -1.242752194404602, + "logps/chosen": -384.41534423828125, + "logps/rejected": -458.1105041503906, + "loss": 0.5235, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.8568652868270874, + "rewards/margins": 0.8491514325141907, + "rewards/rejected": -2.706017017364502, + "step": 8640 + }, + { + "epoch": 0.57, + "learning_rate": 2.3610353454771355e-06, + "logits/chosen": -1.046150803565979, + "logits/rejected": -0.9059313535690308, + "logps/chosen": -342.5575866699219, + "logps/rejected": -422.2818908691406, + "loss": 0.5636, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.5261014699935913, + "rewards/margins": 0.960188090801239, + "rewards/rejected": -2.4862895011901855, + "step": 8650 + }, + { + "epoch": 0.57, + "learning_rate": 2.355334630606467e-06, + "logits/chosen": -1.5204510688781738, + "logits/rejected": -1.2104465961456299, + "logps/chosen": -435.4972229003906, + "logps/rejected": -462.814453125, + "loss": 0.5038, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9520695209503174, + "rewards/margins": 0.9086283445358276, + "rewards/rejected": -2.8606975078582764, + "step": 8660 + }, + { + "epoch": 0.57, + "learning_rate": 2.349634670381231e-06, + "logits/chosen": -1.0414966344833374, + "logits/rejected": -0.9392479658126831, + "logps/chosen": -379.5929260253906, + "logps/rejected": -483.87969970703125, + "loss": 0.5311, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7239383459091187, + "rewards/margins": 0.9448808431625366, + "rewards/rejected": -2.668818950653076, + "step": 8670 + }, + { + "epoch": 0.57, + "learning_rate": 2.3439354945352104e-06, + "logits/chosen": -1.3489819765090942, + "logits/rejected": -1.2225961685180664, + "logps/chosen": -402.8128967285156, + "logps/rejected": -424.5769958496094, + "loss": 0.5713, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.5928926467895508, + "rewards/margins": 0.6616193056106567, + "rewards/rejected": -2.254511594772339, + "step": 8680 + }, + { + "epoch": 0.57, + "learning_rate": 2.3382371327981e-06, + "logits/chosen": -1.326878547668457, + "logits/rejected": -1.2365591526031494, + "logps/chosen": -360.7772216796875, + "logps/rejected": -470.10174560546875, + "loss": 0.4126, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.293009638786316, + "rewards/margins": 1.2361931800842285, + "rewards/rejected": -2.529202938079834, + "step": 8690 + }, + { + "epoch": 0.57, + "learning_rate": 2.3325396148953456e-06, + "logits/chosen": -1.0092710256576538, + "logits/rejected": -1.130898356437683, + "logps/chosen": -336.06768798828125, + "logps/rejected": -493.4744567871094, + "loss": 0.4881, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6476625204086304, + "rewards/margins": 1.0327650308609009, + "rewards/rejected": -2.6804277896881104, + "step": 8700 + }, + { + "epoch": 0.57, + "eval_logits/chosen": -1.238073468208313, + "eval_logits/rejected": -1.0995856523513794, + "eval_logps/chosen": -391.00653076171875, + "eval_logps/rejected": -475.3843688964844, + "eval_loss": 0.4980272352695465, + "eval_rewards/accuracies": 0.6620000004768372, + "eval_rewards/chosen": -1.5900156497955322, + "eval_rewards/margins": 1.0477092266082764, + "eval_rewards/rejected": -2.6377248764038086, + "eval_runtime": 732.2298, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.366, + "step": 8700 + }, + { + "epoch": 0.57, + "learning_rate": 2.3268429705479915e-06, + "logits/chosen": -1.59964120388031, + "logits/rejected": -1.1946229934692383, + "logps/chosen": -376.1258850097656, + "logps/rejected": -466.59326171875, + "loss": 0.3968, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.5131326913833618, + "rewards/margins": 1.3178640604019165, + "rewards/rejected": -2.8309967517852783, + "step": 8710 + }, + { + "epoch": 0.57, + "learning_rate": 2.3211472294725248e-06, + "logits/chosen": -1.1597530841827393, + "logits/rejected": -1.0330088138580322, + "logps/chosen": -386.917724609375, + "logps/rejected": -511.97967529296875, + "loss": 0.4025, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7050254344940186, + "rewards/margins": 1.377290964126587, + "rewards/rejected": -3.0823163986206055, + "step": 8720 + }, + { + "epoch": 0.57, + "learning_rate": 2.315452421380721e-06, + "logits/chosen": -0.9799795150756836, + "logits/rejected": -0.7830728888511658, + "logps/chosen": -428.2682189941406, + "logps/rejected": -490.6246643066406, + "loss": 0.4924, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7332391738891602, + "rewards/margins": 1.1034952402114868, + "rewards/rejected": -2.8367342948913574, + "step": 8730 + }, + { + "epoch": 0.57, + "learning_rate": 2.3097585759794886e-06, + "logits/chosen": -1.045185923576355, + "logits/rejected": -0.8838065266609192, + "logps/chosen": -425.63525390625, + "logps/rejected": -519.7047119140625, + "loss": 0.4465, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.7129396200180054, + "rewards/margins": 1.5746146440505981, + "rewards/rejected": -3.2875542640686035, + "step": 8740 + }, + { + "epoch": 0.57, + "learning_rate": 2.3040657229707155e-06, + "logits/chosen": -1.2783961296081543, + "logits/rejected": -1.0873384475708008, + "logps/chosen": -344.01971435546875, + "logps/rejected": -494.1952209472656, + "loss": 0.4509, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7197210788726807, + "rewards/margins": 1.3861995935440063, + "rewards/rejected": -3.1059205532073975, + "step": 8750 + }, + { + "epoch": 0.57, + "learning_rate": 2.2983738920511104e-06, + "logits/chosen": -1.1936768293380737, + "logits/rejected": -0.9711049795150757, + "logps/chosen": -454.77264404296875, + "logps/rejected": -515.8905029296875, + "loss": 0.4698, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8497978448867798, + "rewards/margins": 1.1141988039016724, + "rewards/rejected": -2.963996410369873, + "step": 8760 + }, + { + "epoch": 0.57, + "learning_rate": 2.2926831129120523e-06, + "logits/chosen": -0.8634411692619324, + "logits/rejected": -0.7416626214981079, + "logps/chosen": -443.77471923828125, + "logps/rejected": -498.29913330078125, + "loss": 0.4592, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0798306465148926, + "rewards/margins": 0.828801155090332, + "rewards/rejected": -2.9086318016052246, + "step": 8770 + }, + { + "epoch": 0.57, + "learning_rate": 2.2869934152394323e-06, + "logits/chosen": -1.1402966976165771, + "logits/rejected": -1.0761340856552124, + "logps/chosen": -481.0205993652344, + "logps/rejected": -516.50146484375, + "loss": 0.5834, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.120551109313965, + "rewards/margins": 0.9108297228813171, + "rewards/rejected": -3.0313808917999268, + "step": 8780 + }, + { + "epoch": 0.58, + "learning_rate": 2.281304828713501e-06, + "logits/chosen": -1.2651450634002686, + "logits/rejected": -1.0738184452056885, + "logps/chosen": -463.01043701171875, + "logps/rejected": -549.8284912109375, + "loss": 0.5532, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.300232172012329, + "rewards/margins": 0.9432049989700317, + "rewards/rejected": -3.2434372901916504, + "step": 8790 + }, + { + "epoch": 0.58, + "learning_rate": 2.275617383008711e-06, + "logits/chosen": -1.1871647834777832, + "logits/rejected": -1.1233766078948975, + "logps/chosen": -420.621337890625, + "logps/rejected": -512.9608154296875, + "loss": 0.4813, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.8513391017913818, + "rewards/margins": 0.950072169303894, + "rewards/rejected": -2.8014113903045654, + "step": 8800 + }, + { + "epoch": 0.58, + "eval_logits/chosen": -1.149058222770691, + "eval_logits/rejected": -1.0135620832443237, + "eval_logps/chosen": -418.1949157714844, + "eval_logps/rejected": -509.9336242675781, + "eval_loss": 0.49589085578918457, + "eval_rewards/accuracies": 0.6620000004768372, + "eval_rewards/chosen": -1.861899495124817, + "eval_rewards/margins": 1.121317744255066, + "eval_rewards/rejected": -2.9832170009613037, + "eval_runtime": 732.1104, + "eval_samples_per_second": 2.732, + "eval_steps_per_second": 1.366, + "step": 8800 + }, + { + "epoch": 0.58, + "learning_rate": 2.269931107793567e-06, + "logits/chosen": -0.9232932925224304, + "logits/rejected": -0.8948391079902649, + "logps/chosen": -370.1566162109375, + "logps/rejected": -467.27716064453125, + "loss": 0.5171, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.5787628889083862, + "rewards/margins": 0.8902810215950012, + "rewards/rejected": -2.4690439701080322, + "step": 8810 + }, + { + "epoch": 0.58, + "learning_rate": 2.2642460327304655e-06, + "logits/chosen": -1.26815927028656, + "logits/rejected": -1.2135612964630127, + "logps/chosen": -416.7547302246094, + "logps/rejected": -502.16326904296875, + "loss": 0.5601, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.721949577331543, + "rewards/margins": 1.0125608444213867, + "rewards/rejected": -2.734510660171509, + "step": 8820 + }, + { + "epoch": 0.58, + "learning_rate": 2.258562187475543e-06, + "logits/chosen": -1.1809900999069214, + "logits/rejected": -0.848108172416687, + "logps/chosen": -405.0650634765625, + "logps/rejected": -456.2456970214844, + "loss": 0.5569, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7811815738677979, + "rewards/margins": 0.8869251012802124, + "rewards/rejected": -2.6681065559387207, + "step": 8830 + }, + { + "epoch": 0.58, + "learning_rate": 2.2528796016785196e-06, + "logits/chosen": -0.9581031799316406, + "logits/rejected": -0.9923402667045593, + "logps/chosen": -365.490966796875, + "logps/rejected": -495.0604553222656, + "loss": 0.4604, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.755730390548706, + "rewards/margins": 1.1990342140197754, + "rewards/rejected": -2.9547646045684814, + "step": 8840 + }, + { + "epoch": 0.58, + "learning_rate": 2.247198304982548e-06, + "logits/chosen": -1.0048309564590454, + "logits/rejected": -0.9457969665527344, + "logps/chosen": -321.87359619140625, + "logps/rejected": -413.46710205078125, + "loss": 0.4841, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5990204811096191, + "rewards/margins": 0.9554295539855957, + "rewards/rejected": -2.554450273513794, + "step": 8850 + }, + { + "epoch": 0.58, + "learning_rate": 2.2415183270240533e-06, + "logits/chosen": -1.4782264232635498, + "logits/rejected": -1.3399109840393066, + "logps/chosen": -394.8039855957031, + "logps/rejected": -499.06524658203125, + "loss": 0.5049, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9547436237335205, + "rewards/margins": 0.9995563626289368, + "rewards/rejected": -2.9542996883392334, + "step": 8860 + }, + { + "epoch": 0.58, + "learning_rate": 2.2358396974325837e-06, + "logits/chosen": -1.071343183517456, + "logits/rejected": -1.0277284383773804, + "logps/chosen": -408.5313720703125, + "logps/rejected": -500.9320373535156, + "loss": 0.4677, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.662637710571289, + "rewards/margins": 1.18667733669281, + "rewards/rejected": -2.8493151664733887, + "step": 8870 + }, + { + "epoch": 0.58, + "learning_rate": 2.2301624458306525e-06, + "logits/chosen": -1.1136915683746338, + "logits/rejected": -1.0989829301834106, + "logps/chosen": -490.1780700683594, + "logps/rejected": -535.048095703125, + "loss": 0.5163, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.308300018310547, + "rewards/margins": 0.9621118307113647, + "rewards/rejected": -3.270411968231201, + "step": 8880 + }, + { + "epoch": 0.58, + "learning_rate": 2.2244866018335855e-06, + "logits/chosen": -1.0626459121704102, + "logits/rejected": -1.003996729850769, + "logps/chosen": -422.763671875, + "logps/rejected": -497.86285400390625, + "loss": 0.6432, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.0570597648620605, + "rewards/margins": 0.6069216728210449, + "rewards/rejected": -2.6639814376831055, + "step": 8890 + }, + { + "epoch": 0.58, + "learning_rate": 2.2188121950493648e-06, + "logits/chosen": -1.2248162031173706, + "logits/rejected": -0.8637872934341431, + "logps/chosen": -421.6446838378906, + "logps/rejected": -421.966552734375, + "loss": 0.535, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0290122032165527, + "rewards/margins": 0.7857424020767212, + "rewards/rejected": -2.8147549629211426, + "step": 8900 + }, + { + "epoch": 0.58, + "eval_logits/chosen": -1.0819333791732788, + "eval_logits/rejected": -0.9508689641952515, + "eval_logps/chosen": -436.3647766113281, + "eval_logps/rejected": -526.4249267578125, + "eval_loss": 0.4916088879108429, + "eval_rewards/accuracies": 0.6660000085830688, + "eval_rewards/chosen": -2.043597936630249, + "eval_rewards/margins": 1.1045318841934204, + "eval_rewards/rejected": -3.148129940032959, + "eval_runtime": 732.6366, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 8900 + }, + { + "epoch": 0.58, + "learning_rate": 2.2131392550784766e-06, + "logits/chosen": -1.1720525026321411, + "logits/rejected": -0.8352081179618835, + "logps/chosen": -497.7474670410156, + "logps/rejected": -506.4822692871094, + "loss": 0.4338, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.128687620162964, + "rewards/margins": 1.0429903268814087, + "rewards/rejected": -3.171678066253662, + "step": 8910 + }, + { + "epoch": 0.58, + "learning_rate": 2.2074678115137533e-06, + "logits/chosen": -1.1111485958099365, + "logits/rejected": -0.9204902648925781, + "logps/chosen": -370.63104248046875, + "logps/rejected": -509.78564453125, + "loss": 0.4269, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.759141206741333, + "rewards/margins": 1.3423799276351929, + "rewards/rejected": -3.1015210151672363, + "step": 8920 + }, + { + "epoch": 0.58, + "learning_rate": 2.201797893940224e-06, + "logits/chosen": -0.8966342806816101, + "logits/rejected": -0.9461180567741394, + "logps/chosen": -429.45947265625, + "logps/rejected": -564.6712646484375, + "loss": 0.4905, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.962821364402771, + "rewards/margins": 1.148612380027771, + "rewards/rejected": -3.111433267593384, + "step": 8930 + }, + { + "epoch": 0.58, + "learning_rate": 2.196129531934956e-06, + "logits/chosen": -0.9999095797538757, + "logits/rejected": -0.9462803602218628, + "logps/chosen": -408.2229919433594, + "logps/rejected": -495.52923583984375, + "loss": 0.5563, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.703089714050293, + "rewards/margins": 1.0536882877349854, + "rewards/rejected": -2.7567780017852783, + "step": 8940 + }, + { + "epoch": 0.59, + "learning_rate": 2.190462755066902e-06, + "logits/chosen": -1.1943506002426147, + "logits/rejected": -0.9096410870552063, + "logps/chosen": -478.9681091308594, + "logps/rejected": -559.5634155273438, + "loss": 0.4374, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1656689643859863, + "rewards/margins": 1.0608609914779663, + "rewards/rejected": -3.2265305519104004, + "step": 8950 + }, + { + "epoch": 0.59, + "learning_rate": 2.184797592896746e-06, + "logits/chosen": -1.2017793655395508, + "logits/rejected": -1.08447265625, + "logps/chosen": -423.525146484375, + "logps/rejected": -497.7538146972656, + "loss": 0.5069, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8908952474594116, + "rewards/margins": 1.0076181888580322, + "rewards/rejected": -2.8985133171081543, + "step": 8960 + }, + { + "epoch": 0.59, + "learning_rate": 2.17913407497675e-06, + "logits/chosen": -1.0965301990509033, + "logits/rejected": -1.0276085138320923, + "logps/chosen": -331.51068115234375, + "logps/rejected": -470.09912109375, + "loss": 0.4991, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.5312869548797607, + "rewards/margins": 1.0463721752166748, + "rewards/rejected": -2.5776591300964355, + "step": 8970 + }, + { + "epoch": 0.59, + "learning_rate": 2.173472230850596e-06, + "logits/chosen": -1.2828447818756104, + "logits/rejected": -0.9710990786552429, + "logps/chosen": -350.43133544921875, + "logps/rejected": -389.3118591308594, + "loss": 0.5289, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.5016876459121704, + "rewards/margins": 0.7932427525520325, + "rewards/rejected": -2.2949302196502686, + "step": 8980 + }, + { + "epoch": 0.59, + "learning_rate": 2.1678120900532375e-06, + "logits/chosen": -1.0349891185760498, + "logits/rejected": -0.8604212999343872, + "logps/chosen": -460.71142578125, + "logps/rejected": -554.3140869140625, + "loss": 0.4128, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.2479658126831055, + "rewards/margins": 1.2286293506622314, + "rewards/rejected": -3.476595401763916, + "step": 8990 + }, + { + "epoch": 0.59, + "learning_rate": 2.1621536821107412e-06, + "logits/chosen": -1.0512446165084839, + "logits/rejected": -0.8796029090881348, + "logps/chosen": -385.85906982421875, + "logps/rejected": -437.2776794433594, + "loss": 0.5399, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8328920602798462, + "rewards/margins": 0.9345098733901978, + "rewards/rejected": -2.767401933670044, + "step": 9000 + }, + { + "epoch": 0.59, + "eval_logits/chosen": -1.0397822856903076, + "eval_logits/rejected": -0.9097519516944885, + "eval_logps/chosen": -422.9480895996094, + "eval_logps/rejected": -515.3348999023438, + "eval_loss": 0.4937884509563446, + "eval_rewards/accuracies": 0.6629999876022339, + "eval_rewards/chosen": -1.9094310998916626, + "eval_rewards/margins": 1.1277989149093628, + "eval_rewards/rejected": -3.0372297763824463, + "eval_runtime": 731.1211, + "eval_samples_per_second": 2.736, + "eval_steps_per_second": 1.368, + "step": 9000 + }, + { + "epoch": 0.59, + "learning_rate": 2.1564970365401346e-06, + "logits/chosen": -1.2500946521759033, + "logits/rejected": -0.9484453201293945, + "logps/chosen": -394.15130615234375, + "logps/rejected": -437.1795349121094, + "loss": 0.6583, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.1000022888183594, + "rewards/margins": 0.8103631138801575, + "rewards/rejected": -2.910365581512451, + "step": 9010 + }, + { + "epoch": 0.59, + "learning_rate": 2.1508421828492527e-06, + "logits/chosen": -1.31488037109375, + "logits/rejected": -1.0038257837295532, + "logps/chosen": -388.72802734375, + "logps/rejected": -420.581298828125, + "loss": 0.5326, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.6353528499603271, + "rewards/margins": 1.0009357929229736, + "rewards/rejected": -2.636288642883301, + "step": 9020 + }, + { + "epoch": 0.59, + "learning_rate": 2.145189150536582e-06, + "logits/chosen": -1.0028223991394043, + "logits/rejected": -0.9240818023681641, + "logps/chosen": -405.3058776855469, + "logps/rejected": -448.379150390625, + "loss": 0.5105, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8313636779785156, + "rewards/margins": 0.9247555732727051, + "rewards/rejected": -2.7561192512512207, + "step": 9030 + }, + { + "epoch": 0.59, + "learning_rate": 2.139537969091107e-06, + "logits/chosen": -0.8752814531326294, + "logits/rejected": -0.8566150665283203, + "logps/chosen": -435.94989013671875, + "logps/rejected": -458.0609436035156, + "loss": 0.5249, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7036616802215576, + "rewards/margins": 0.8428160548210144, + "rewards/rejected": -2.546477794647217, + "step": 9040 + }, + { + "epoch": 0.59, + "learning_rate": 2.1338886679921603e-06, + "logits/chosen": -1.0392322540283203, + "logits/rejected": -0.9704221487045288, + "logps/chosen": -401.290771484375, + "logps/rejected": -473.9989318847656, + "loss": 0.4822, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.5974773168563843, + "rewards/margins": 0.9311855435371399, + "rewards/rejected": -2.528662919998169, + "step": 9050 + }, + { + "epoch": 0.59, + "learning_rate": 2.128241276709263e-06, + "logits/chosen": -1.2574546337127686, + "logits/rejected": -1.3090790510177612, + "logps/chosen": -331.599853515625, + "logps/rejected": -454.751953125, + "loss": 0.4899, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.2833807468414307, + "rewards/margins": 1.0311470031738281, + "rewards/rejected": -2.3145275115966797, + "step": 9060 + }, + { + "epoch": 0.59, + "learning_rate": 2.1225958247019746e-06, + "logits/chosen": -1.4055688381195068, + "logits/rejected": -1.432252049446106, + "logps/chosen": -336.8355407714844, + "logps/rejected": -445.4163513183594, + "loss": 0.4993, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.4969367980957031, + "rewards/margins": 0.8844121098518372, + "rewards/rejected": -2.3813490867614746, + "step": 9070 + }, + { + "epoch": 0.59, + "learning_rate": 2.1169523414197383e-06, + "logits/chosen": -1.077289342880249, + "logits/rejected": -1.0037438869476318, + "logps/chosen": -342.13739013671875, + "logps/rejected": -416.0020446777344, + "loss": 0.6481, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -1.4333170652389526, + "rewards/margins": 0.505429744720459, + "rewards/rejected": -1.938746690750122, + "step": 9080 + }, + { + "epoch": 0.59, + "learning_rate": 2.1113108563017267e-06, + "logits/chosen": -1.0560085773468018, + "logits/rejected": -1.0217021703720093, + "logps/chosen": -385.6211242675781, + "logps/rejected": -463.80078125, + "loss": 0.5424, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7283786535263062, + "rewards/margins": 1.0880886316299438, + "rewards/rejected": -2.81646728515625, + "step": 9090 + }, + { + "epoch": 0.6, + "learning_rate": 2.1056713987766905e-06, + "logits/chosen": -1.2946921586990356, + "logits/rejected": -1.1029138565063477, + "logps/chosen": -377.98126220703125, + "logps/rejected": -428.6024475097656, + "loss": 0.512, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.6278095245361328, + "rewards/margins": 0.9933661222457886, + "rewards/rejected": -2.62117600440979, + "step": 9100 + }, + { + "epoch": 0.6, + "eval_logits/chosen": -1.2002038955688477, + "eval_logits/rejected": -1.0658358335494995, + "eval_logps/chosen": -383.3267517089844, + "eval_logps/rejected": -461.37103271484375, + "eval_loss": 0.4937075674533844, + "eval_rewards/accuracies": 0.6729999780654907, + "eval_rewards/chosen": -1.5132176876068115, + "eval_rewards/margins": 0.9843739867210388, + "eval_rewards/rejected": -2.497591495513916, + "eval_runtime": 733.2161, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 9100 + }, + { + "epoch": 0.6, + "learning_rate": 2.1000339982628022e-06, + "logits/chosen": -1.0017447471618652, + "logits/rejected": -0.913290798664093, + "logps/chosen": -436.9093322753906, + "logps/rejected": -467.470947265625, + "loss": 0.5786, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.8563644886016846, + "rewards/margins": 0.6488175988197327, + "rewards/rejected": -2.5051820278167725, + "step": 9110 + }, + { + "epoch": 0.6, + "learning_rate": 2.0943986841675043e-06, + "logits/chosen": -1.2511661052703857, + "logits/rejected": -0.9566980600357056, + "logps/chosen": -357.22930908203125, + "logps/rejected": -419.00506591796875, + "loss": 0.54, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.540183186531067, + "rewards/margins": 0.8111560940742493, + "rewards/rejected": -2.351339340209961, + "step": 9120 + }, + { + "epoch": 0.6, + "learning_rate": 2.088765485887356e-06, + "logits/chosen": -1.2423206567764282, + "logits/rejected": -0.9856125116348267, + "logps/chosen": -393.86920166015625, + "logps/rejected": -431.34765625, + "loss": 0.5275, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.491542935371399, + "rewards/margins": 0.7910388112068176, + "rewards/rejected": -2.2825820446014404, + "step": 9130 + }, + { + "epoch": 0.6, + "learning_rate": 2.083134432807879e-06, + "logits/chosen": -1.2834888696670532, + "logits/rejected": -1.0755784511566162, + "logps/chosen": -349.40350341796875, + "logps/rejected": -488.70391845703125, + "loss": 0.5207, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.551313877105713, + "rewards/margins": 1.1778234243392944, + "rewards/rejected": -2.729137420654297, + "step": 9140 + }, + { + "epoch": 0.6, + "learning_rate": 2.077505554303404e-06, + "logits/chosen": -1.0973273515701294, + "logits/rejected": -1.085224986076355, + "logps/chosen": -307.36785888671875, + "logps/rejected": -404.21368408203125, + "loss": 0.4621, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.3281586170196533, + "rewards/margins": 0.9247921705245972, + "rewards/rejected": -2.252951145172119, + "step": 9150 + }, + { + "epoch": 0.6, + "learning_rate": 2.071878879736918e-06, + "logits/chosen": -1.23152756690979, + "logits/rejected": -1.0925686359405518, + "logps/chosen": -442.94647216796875, + "logps/rejected": -604.2838134765625, + "loss": 0.5539, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.961069107055664, + "rewards/margins": 0.8305347561836243, + "rewards/rejected": -2.7916038036346436, + "step": 9160 + }, + { + "epoch": 0.6, + "learning_rate": 2.0662544384599136e-06, + "logits/chosen": -1.0271787643432617, + "logits/rejected": -0.8980680704116821, + "logps/chosen": -358.5318908691406, + "logps/rejected": -441.77337646484375, + "loss": 0.5172, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.5642528533935547, + "rewards/margins": 0.9972681999206543, + "rewards/rejected": -2.56152081489563, + "step": 9170 + }, + { + "epoch": 0.6, + "learning_rate": 2.0606322598122314e-06, + "logits/chosen": -0.9368621110916138, + "logits/rejected": -1.0792651176452637, + "logps/chosen": -381.1671447753906, + "logps/rejected": -444.1542053222656, + "loss": 0.6215, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.9465354681015015, + "rewards/margins": 0.43326178193092346, + "rewards/rejected": -2.3797974586486816, + "step": 9180 + }, + { + "epoch": 0.6, + "learning_rate": 2.0550123731219085e-06, + "logits/chosen": -1.480641484260559, + "logits/rejected": -1.0889581441879272, + "logps/chosen": -416.18988037109375, + "logps/rejected": -478.01580810546875, + "loss": 0.4067, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6301004886627197, + "rewards/margins": 1.029707670211792, + "rewards/rejected": -2.659808397293091, + "step": 9190 + }, + { + "epoch": 0.6, + "learning_rate": 2.0493948077050267e-06, + "logits/chosen": -0.734448254108429, + "logits/rejected": -0.6183160543441772, + "logps/chosen": -385.55157470703125, + "logps/rejected": -468.9554748535156, + "loss": 0.5069, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9229555130004883, + "rewards/margins": 1.0149352550506592, + "rewards/rejected": -2.9378905296325684, + "step": 9200 + }, + { + "epoch": 0.6, + "eval_logits/chosen": -1.0345836877822876, + "eval_logits/rejected": -0.9101244211196899, + "eval_logps/chosen": -411.07568359375, + "eval_logps/rejected": -487.1391906738281, + "eval_loss": 0.49305132031440735, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -1.7907072305679321, + "eval_rewards/margins": 0.9645655751228333, + "eval_rewards/rejected": -2.755272626876831, + "eval_runtime": 733.6293, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 9200 + }, + { + "epoch": 0.6, + "learning_rate": 2.0437795928655596e-06, + "logits/chosen": -1.1340954303741455, + "logits/rejected": -1.1976003646850586, + "logps/chosen": -471.8667907714844, + "logps/rejected": -520.8145751953125, + "loss": 0.549, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.9015413522720337, + "rewards/margins": 0.6713147759437561, + "rewards/rejected": -2.5728561878204346, + "step": 9210 + }, + { + "epoch": 0.6, + "learning_rate": 2.0381667578952184e-06, + "logits/chosen": -1.1571053266525269, + "logits/rejected": -1.022148609161377, + "logps/chosen": -411.462646484375, + "logps/rejected": -526.1029052734375, + "loss": 0.4672, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.0058977603912354, + "rewards/margins": 1.0902332067489624, + "rewards/rejected": -3.0961310863494873, + "step": 9220 + }, + { + "epoch": 0.6, + "learning_rate": 2.0325563320732995e-06, + "logits/chosen": -1.2341101169586182, + "logits/rejected": -1.111156702041626, + "logps/chosen": -445.73834228515625, + "logps/rejected": -505.947021484375, + "loss": 0.4702, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.8240001201629639, + "rewards/margins": 1.042265772819519, + "rewards/rejected": -2.8662657737731934, + "step": 9230 + }, + { + "epoch": 0.6, + "learning_rate": 2.026948344666532e-06, + "logits/chosen": -0.911036491394043, + "logits/rejected": -1.0102976560592651, + "logps/chosen": -382.86279296875, + "logps/rejected": -468.21502685546875, + "loss": 0.5338, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.837368369102478, + "rewards/margins": 0.8579694032669067, + "rewards/rejected": -2.6953375339508057, + "step": 9240 + }, + { + "epoch": 0.61, + "learning_rate": 2.0213428249289257e-06, + "logits/chosen": -0.6779653429985046, + "logits/rejected": -0.8694950938224792, + "logps/chosen": -363.29046630859375, + "logps/rejected": -479.02117919921875, + "loss": 0.4412, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6474993228912354, + "rewards/margins": 1.1392320394515991, + "rewards/rejected": -2.786731243133545, + "step": 9250 + }, + { + "epoch": 0.61, + "learning_rate": 2.0157398021016175e-06, + "logits/chosen": -0.9124547839164734, + "logits/rejected": -0.9362983703613281, + "logps/chosen": -314.9878845214844, + "logps/rejected": -436.501220703125, + "loss": 0.5809, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.6745256185531616, + "rewards/margins": 0.7530856132507324, + "rewards/rejected": -2.4276108741760254, + "step": 9260 + }, + { + "epoch": 0.61, + "learning_rate": 2.010139305412719e-06, + "logits/chosen": -1.4748836755752563, + "logits/rejected": -1.1832959651947021, + "logps/chosen": -457.10833740234375, + "logps/rejected": -528.8362426757812, + "loss": 0.4254, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8051267862319946, + "rewards/margins": 1.1409196853637695, + "rewards/rejected": -2.9460463523864746, + "step": 9270 + }, + { + "epoch": 0.61, + "learning_rate": 2.0045413640771644e-06, + "logits/chosen": -1.121006965637207, + "logits/rejected": -0.9204033017158508, + "logps/chosen": -465.58013916015625, + "logps/rejected": -592.5983276367188, + "loss": 0.4731, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1041476726531982, + "rewards/margins": 1.2376271486282349, + "rewards/rejected": -3.3417747020721436, + "step": 9280 + }, + { + "epoch": 0.61, + "learning_rate": 1.998946007296558e-06, + "logits/chosen": -1.1622593402862549, + "logits/rejected": -1.052321195602417, + "logps/chosen": -503.66455078125, + "logps/rejected": -544.9689331054688, + "loss": 0.4878, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9506393671035767, + "rewards/margins": 1.0474134683609009, + "rewards/rejected": -2.9980528354644775, + "step": 9290 + }, + { + "epoch": 0.61, + "learning_rate": 1.9933532642590215e-06, + "logits/chosen": -0.801226794719696, + "logits/rejected": -0.5509706735610962, + "logps/chosen": -321.21893310546875, + "logps/rejected": -397.5530090332031, + "loss": 0.4272, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.3206064701080322, + "rewards/margins": 1.2649487257003784, + "rewards/rejected": -2.5855555534362793, + "step": 9300 + }, + { + "epoch": 0.61, + "eval_logits/chosen": -1.0575023889541626, + "eval_logits/rejected": -0.9299944043159485, + "eval_logps/chosen": -413.5278015136719, + "eval_logps/rejected": -500.4742126464844, + "eval_loss": 0.49185654520988464, + "eval_rewards/accuracies": 0.6729999780654907, + "eval_rewards/chosen": -1.8152281045913696, + "eval_rewards/margins": 1.073394775390625, + "eval_rewards/rejected": -2.888622999191284, + "eval_runtime": 731.6322, + "eval_samples_per_second": 2.734, + "eval_steps_per_second": 1.367, + "step": 9300 + }, + { + "epoch": 0.61, + "learning_rate": 1.987763164139042e-06, + "logits/chosen": -1.2010269165039062, + "logits/rejected": -0.964155375957489, + "logps/chosen": -373.0898742675781, + "logps/rejected": -489.07989501953125, + "loss": 0.5161, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6783478260040283, + "rewards/margins": 1.1373569965362549, + "rewards/rejected": -2.815704822540283, + "step": 9310 + }, + { + "epoch": 0.61, + "learning_rate": 1.982175736097321e-06, + "logits/chosen": -0.9910721778869629, + "logits/rejected": -0.963726818561554, + "logps/chosen": -491.3431701660156, + "logps/rejected": -576.5433349609375, + "loss": 0.6689, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.168020725250244, + "rewards/margins": 0.6818044781684875, + "rewards/rejected": -2.849825382232666, + "step": 9320 + }, + { + "epoch": 0.61, + "learning_rate": 1.9765910092806196e-06, + "logits/chosen": -1.0442947149276733, + "logits/rejected": -0.9146569967269897, + "logps/chosen": -322.37860107421875, + "logps/rejected": -365.389892578125, + "loss": 0.5562, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.4430307149887085, + "rewards/margins": 0.6266955137252808, + "rewards/rejected": -2.0697264671325684, + "step": 9330 + }, + { + "epoch": 0.61, + "learning_rate": 1.9710090128216083e-06, + "logits/chosen": -1.1570490598678589, + "logits/rejected": -0.9983047246932983, + "logps/chosen": -426.0396423339844, + "logps/rejected": -540.4515991210938, + "loss": 0.4606, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.0330958366394043, + "rewards/margins": 1.305750846862793, + "rewards/rejected": -3.3388469219207764, + "step": 9340 + }, + { + "epoch": 0.61, + "learning_rate": 1.9654297758387155e-06, + "logits/chosen": -1.0099811553955078, + "logits/rejected": -0.8388859629631042, + "logps/chosen": -346.5732116699219, + "logps/rejected": -471.65362548828125, + "loss": 0.5067, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.857031226158142, + "rewards/margins": 1.0435500144958496, + "rewards/rejected": -2.9005813598632812, + "step": 9350 + }, + { + "epoch": 0.61, + "learning_rate": 1.9598533274359736e-06, + "logits/chosen": -0.9493533968925476, + "logits/rejected": -1.040316104888916, + "logps/chosen": -452.26397705078125, + "logps/rejected": -495.58001708984375, + "loss": 0.7206, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.1258368492126465, + "rewards/margins": 0.3595449924468994, + "rewards/rejected": -2.485381603240967, + "step": 9360 + }, + { + "epoch": 0.61, + "learning_rate": 1.9542796967028697e-06, + "logits/chosen": -1.2399563789367676, + "logits/rejected": -0.9904924631118774, + "logps/chosen": -399.768310546875, + "logps/rejected": -469.8306579589844, + "loss": 0.4607, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8152774572372437, + "rewards/margins": 0.8830522298812866, + "rewards/rejected": -2.6983296871185303, + "step": 9370 + }, + { + "epoch": 0.61, + "learning_rate": 1.948708912714192e-06, + "logits/chosen": -0.6782970428466797, + "logits/rejected": -0.7653140425682068, + "logps/chosen": -454.6788024902344, + "logps/rejected": -513.2496337890625, + "loss": 0.5389, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0773351192474365, + "rewards/margins": 0.8950663805007935, + "rewards/rejected": -2.9724013805389404, + "step": 9380 + }, + { + "epoch": 0.61, + "learning_rate": 1.9431410045298786e-06, + "logits/chosen": -0.7093806862831116, + "logits/rejected": -0.8312298059463501, + "logps/chosen": -424.17620849609375, + "logps/rejected": -498.23980712890625, + "loss": 0.5269, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.0705084800720215, + "rewards/margins": 0.7503930330276489, + "rewards/rejected": -2.820901393890381, + "step": 9390 + }, + { + "epoch": 0.62, + "learning_rate": 1.9375760011948654e-06, + "logits/chosen": -1.0410220623016357, + "logits/rejected": -1.040238857269287, + "logps/chosen": -384.260009765625, + "logps/rejected": -512.1881713867188, + "loss": 0.4398, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.845820665359497, + "rewards/margins": 1.0560771226882935, + "rewards/rejected": -2.901897668838501, + "step": 9400 + }, + { + "epoch": 0.62, + "eval_logits/chosen": -0.9658358693122864, + "eval_logits/rejected": -0.8458749651908875, + "eval_logps/chosen": -438.275634765625, + "eval_logps/rejected": -514.0955810546875, + "eval_loss": 0.49358147382736206, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -2.062706470489502, + "eval_rewards/margins": 0.9621301889419556, + "eval_rewards/rejected": -3.024836778640747, + "eval_runtime": 730.7547, + "eval_samples_per_second": 2.737, + "eval_steps_per_second": 1.368, + "step": 9400 + }, + { + "epoch": 0.62, + "learning_rate": 1.932013931738937e-06, + "logits/chosen": -0.9855095148086548, + "logits/rejected": -0.7755752205848694, + "logps/chosen": -410.9771423339844, + "logps/rejected": -556.1227416992188, + "loss": 0.3902, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.073754072189331, + "rewards/margins": 1.322638750076294, + "rewards/rejected": -3.396392822265625, + "step": 9410 + }, + { + "epoch": 0.62, + "learning_rate": 1.9264548251765717e-06, + "logits/chosen": -1.0908194780349731, + "logits/rejected": -1.0226426124572754, + "logps/chosen": -389.06085205078125, + "logps/rejected": -518.1353759765625, + "loss": 0.3952, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.8525817394256592, + "rewards/margins": 1.3218623399734497, + "rewards/rejected": -3.1744441986083984, + "step": 9420 + }, + { + "epoch": 0.62, + "learning_rate": 1.9208987105067924e-06, + "logits/chosen": -0.7906646132469177, + "logits/rejected": -0.6241647601127625, + "logps/chosen": -440.5582580566406, + "logps/rejected": -484.5211486816406, + "loss": 0.6315, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.260011911392212, + "rewards/margins": 0.6785053610801697, + "rewards/rejected": -2.9385170936584473, + "step": 9430 + }, + { + "epoch": 0.62, + "learning_rate": 1.9153456167130154e-06, + "logits/chosen": -0.8590573072433472, + "logits/rejected": -0.8843520283699036, + "logps/chosen": -428.5514221191406, + "logps/rejected": -539.2299194335938, + "loss": 0.4938, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2448079586029053, + "rewards/margins": 0.8447982668876648, + "rewards/rejected": -3.0896060466766357, + "step": 9440 + }, + { + "epoch": 0.62, + "learning_rate": 1.9097955727628975e-06, + "logits/chosen": -1.049441933631897, + "logits/rejected": -1.0770914554595947, + "logps/chosen": -410.7098693847656, + "logps/rejected": -502.4935607910156, + "loss": 0.5324, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.1472437381744385, + "rewards/margins": 0.7984870672225952, + "rewards/rejected": -2.945730686187744, + "step": 9450 + }, + { + "epoch": 0.62, + "learning_rate": 1.904248607608187e-06, + "logits/chosen": -0.6168997883796692, + "logits/rejected": -0.8384846448898315, + "logps/chosen": -466.0945739746094, + "logps/rejected": -519.4091186523438, + "loss": 0.4144, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.0887458324432373, + "rewards/margins": 1.004212737083435, + "rewards/rejected": -3.092958927154541, + "step": 9460 + }, + { + "epoch": 0.62, + "learning_rate": 1.8987047501845714e-06, + "logits/chosen": -0.9473953247070312, + "logits/rejected": -0.7209688425064087, + "logps/chosen": -393.9309997558594, + "logps/rejected": -504.14508056640625, + "loss": 0.4709, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.2700653076171875, + "rewards/margins": 1.1645954847335815, + "rewards/rejected": -3.4346611499786377, + "step": 9470 + }, + { + "epoch": 0.62, + "learning_rate": 1.8931640294115267e-06, + "logits/chosen": -0.6625471115112305, + "logits/rejected": -0.43366652727127075, + "logps/chosen": -418.9068298339844, + "logps/rejected": -512.6217651367188, + "loss": 0.4559, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.249008893966675, + "rewards/margins": 1.0924519300460815, + "rewards/rejected": -3.341460704803467, + "step": 9480 + }, + { + "epoch": 0.62, + "learning_rate": 1.8876264741921662e-06, + "logits/chosen": -0.7319644689559937, + "logits/rejected": -0.6956400871276855, + "logps/chosen": -423.14947509765625, + "logps/rejected": -560.1439208984375, + "loss": 0.4691, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.330239772796631, + "rewards/margins": 1.4242993593215942, + "rewards/rejected": -3.7545394897460938, + "step": 9490 + }, + { + "epoch": 0.62, + "learning_rate": 1.8820921134130912e-06, + "logits/chosen": -0.9113744497299194, + "logits/rejected": -0.5819921493530273, + "logps/chosen": -476.8115234375, + "logps/rejected": -583.10400390625, + "loss": 0.498, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.462993621826172, + "rewards/margins": 1.5120172500610352, + "rewards/rejected": -3.975010633468628, + "step": 9500 + }, + { + "epoch": 0.62, + "eval_logits/chosen": -0.7636699676513672, + "eval_logits/rejected": -0.6522924304008484, + "eval_logps/chosen": -485.16644287109375, + "eval_logps/rejected": -572.1414184570312, + "eval_loss": 0.49296244978904724, + "eval_rewards/accuracies": 0.6644999980926514, + "eval_rewards/chosen": -2.5316150188446045, + "eval_rewards/margins": 1.07368004322052, + "eval_rewards/rejected": -3.605294942855835, + "eval_runtime": 733.6355, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 9500 + }, + { + "epoch": 0.62, + "learning_rate": 1.8765609759442378e-06, + "logits/chosen": -0.34705209732055664, + "logits/rejected": -0.4685601592063904, + "logps/chosen": -509.03216552734375, + "logps/rejected": -588.2866821289062, + "loss": 0.4986, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.6771090030670166, + "rewards/margins": 0.9006662368774414, + "rewards/rejected": -3.577775239944458, + "step": 9510 + }, + { + "epoch": 0.62, + "learning_rate": 1.8710330906387288e-06, + "logits/chosen": -1.2283780574798584, + "logits/rejected": -1.1918247938156128, + "logps/chosen": -496.95623779296875, + "logps/rejected": -614.5147094726562, + "loss": 0.4733, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.584685802459717, + "rewards/margins": 0.9396859407424927, + "rewards/rejected": -3.52437162399292, + "step": 9520 + }, + { + "epoch": 0.62, + "learning_rate": 1.8655084863327222e-06, + "logits/chosen": -0.8194085955619812, + "logits/rejected": -0.7681763768196106, + "logps/chosen": -369.12579345703125, + "logps/rejected": -436.62646484375, + "loss": 0.6334, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -1.8566782474517822, + "rewards/margins": 0.6133848428726196, + "rewards/rejected": -2.4700629711151123, + "step": 9530 + }, + { + "epoch": 0.62, + "learning_rate": 1.8599871918452603e-06, + "logits/chosen": -0.7199943661689758, + "logits/rejected": -0.789899468421936, + "logps/chosen": -427.11700439453125, + "logps/rejected": -527.3338012695312, + "loss": 0.5339, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.0402514934539795, + "rewards/margins": 0.8696733713150024, + "rewards/rejected": -2.9099249839782715, + "step": 9540 + }, + { + "epoch": 0.62, + "learning_rate": 1.8544692359781192e-06, + "logits/chosen": -0.7468751072883606, + "logits/rejected": -0.7449430227279663, + "logps/chosen": -369.1566467285156, + "logps/rejected": -454.22149658203125, + "loss": 0.4088, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8157199621200562, + "rewards/margins": 1.1601203680038452, + "rewards/rejected": -2.9758403301239014, + "step": 9550 + }, + { + "epoch": 0.63, + "learning_rate": 1.8489546475156602e-06, + "logits/chosen": -1.0949957370758057, + "logits/rejected": -1.0754151344299316, + "logps/chosen": -448.9339294433594, + "logps/rejected": -527.186279296875, + "loss": 0.5017, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.2944109439849854, + "rewards/margins": 0.9642025828361511, + "rewards/rejected": -3.258613109588623, + "step": 9560 + }, + { + "epoch": 0.63, + "learning_rate": 1.8434434552246778e-06, + "logits/chosen": -0.7982507944107056, + "logits/rejected": -0.7856664657592773, + "logps/chosen": -393.72259521484375, + "logps/rejected": -494.39874267578125, + "loss": 0.4493, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.84091055393219, + "rewards/margins": 1.1193912029266357, + "rewards/rejected": -2.9603018760681152, + "step": 9570 + }, + { + "epoch": 0.63, + "learning_rate": 1.837935687854251e-06, + "logits/chosen": -0.9066284894943237, + "logits/rejected": -0.7065574526786804, + "logps/chosen": -458.378662109375, + "logps/rejected": -556.5958251953125, + "loss": 0.4269, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.368319511413574, + "rewards/margins": 1.3001726865768433, + "rewards/rejected": -3.668492078781128, + "step": 9580 + }, + { + "epoch": 0.63, + "learning_rate": 1.832431374135592e-06, + "logits/chosen": -0.9596296548843384, + "logits/rejected": -1.0166507959365845, + "logps/chosen": -492.32879638671875, + "logps/rejected": -600.0111694335938, + "loss": 0.4765, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.438934564590454, + "rewards/margins": 1.2655547857284546, + "rewards/rejected": -3.7044894695281982, + "step": 9590 + }, + { + "epoch": 0.63, + "learning_rate": 1.8269305427818977e-06, + "logits/chosen": -0.9456634521484375, + "logits/rejected": -0.8710576891899109, + "logps/chosen": -447.292724609375, + "logps/rejected": -526.4971923828125, + "loss": 0.4865, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3867650032043457, + "rewards/margins": 1.0686821937561035, + "rewards/rejected": -3.455446720123291, + "step": 9600 + }, + { + "epoch": 0.63, + "eval_logits/chosen": -0.7693200707435608, + "eval_logits/rejected": -0.6561520099639893, + "eval_logps/chosen": -475.1277770996094, + "eval_logps/rejected": -570.9478759765625, + "eval_loss": 0.4916233420372009, + "eval_rewards/accuracies": 0.6685000061988831, + "eval_rewards/chosen": -2.431227922439575, + "eval_rewards/margins": 1.1621315479278564, + "eval_rewards/rejected": -3.5933594703674316, + "eval_runtime": 733.5856, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 9600 + }, + { + "epoch": 0.63, + "learning_rate": 1.821433222488199e-06, + "logits/chosen": -0.4729815125465393, + "logits/rejected": -0.5782935619354248, + "logps/chosen": -471.65423583984375, + "logps/rejected": -550.3768310546875, + "loss": 0.5554, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.460395097732544, + "rewards/margins": 1.0647014379501343, + "rewards/rejected": -3.5250961780548096, + "step": 9610 + }, + { + "epoch": 0.63, + "learning_rate": 1.8159394419312112e-06, + "logits/chosen": -0.9872452616691589, + "logits/rejected": -0.7170965075492859, + "logps/chosen": -487.39447021484375, + "logps/rejected": -613.5425415039062, + "loss": 0.3922, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.286644458770752, + "rewards/margins": 1.726867914199829, + "rewards/rejected": -4.01351261138916, + "step": 9620 + }, + { + "epoch": 0.63, + "learning_rate": 1.8104492297691845e-06, + "logits/chosen": -0.9232928156852722, + "logits/rejected": -0.7564734220504761, + "logps/chosen": -506.34637451171875, + "logps/rejected": -597.8238525390625, + "loss": 0.5703, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.7982330322265625, + "rewards/margins": 1.1558990478515625, + "rewards/rejected": -3.954132080078125, + "step": 9630 + }, + { + "epoch": 0.63, + "learning_rate": 1.8049626146417562e-06, + "logits/chosen": -0.25446242094039917, + "logits/rejected": -0.4590334892272949, + "logps/chosen": -376.924560546875, + "logps/rejected": -466.033935546875, + "loss": 0.6335, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.2184863090515137, + "rewards/margins": 0.9282231330871582, + "rewards/rejected": -3.146709442138672, + "step": 9640 + }, + { + "epoch": 0.63, + "learning_rate": 1.7994796251697983e-06, + "logits/chosen": -0.6224043965339661, + "logits/rejected": -0.4321642518043518, + "logps/chosen": -441.83721923828125, + "logps/rejected": -622.97216796875, + "loss": 0.448, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.497983455657959, + "rewards/margins": 1.4157010316848755, + "rewards/rejected": -3.913684129714966, + "step": 9650 + }, + { + "epoch": 0.63, + "learning_rate": 1.794000289955269e-06, + "logits/chosen": -0.7555705904960632, + "logits/rejected": -0.8453912734985352, + "logps/chosen": -528.7978515625, + "logps/rejected": -593.1597290039062, + "loss": 0.5633, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.6173758506774902, + "rewards/margins": 0.9580874443054199, + "rewards/rejected": -3.5754635334014893, + "step": 9660 + }, + { + "epoch": 0.63, + "learning_rate": 1.7885246375810646e-06, + "logits/chosen": -0.4721905589103699, + "logits/rejected": -0.4663899540901184, + "logps/chosen": -453.2267150878906, + "logps/rejected": -554.3441162109375, + "loss": 0.4463, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.2895278930664062, + "rewards/margins": 1.0487935543060303, + "rewards/rejected": -3.3383216857910156, + "step": 9670 + }, + { + "epoch": 0.63, + "learning_rate": 1.7830526966108713e-06, + "logits/chosen": -0.6492630243301392, + "logits/rejected": -0.5194262266159058, + "logps/chosen": -447.9353942871094, + "logps/rejected": -573.6887817382812, + "loss": 0.3638, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.6146247386932373, + "rewards/margins": 1.5436484813690186, + "rewards/rejected": -4.158273696899414, + "step": 9680 + }, + { + "epoch": 0.63, + "learning_rate": 1.7775844955890129e-06, + "logits/chosen": -0.6144987940788269, + "logits/rejected": -0.5048569440841675, + "logps/chosen": -426.19696044921875, + "logps/rejected": -570.0430908203125, + "loss": 0.3568, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.1465060710906982, + "rewards/margins": 1.55422842502594, + "rewards/rejected": -3.7007346153259277, + "step": 9690 + }, + { + "epoch": 0.63, + "learning_rate": 1.7721200630403046e-06, + "logits/chosen": -0.5308637022972107, + "logits/rejected": -0.5351754426956177, + "logps/chosen": -431.6376953125, + "logps/rejected": -526.9102172851562, + "loss": 0.5823, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.3587536811828613, + "rewards/margins": 0.6972657442092896, + "rewards/rejected": -3.0560193061828613, + "step": 9700 + }, + { + "epoch": 0.63, + "eval_logits/chosen": -0.7246482968330383, + "eval_logits/rejected": -0.6136430501937866, + "eval_logps/chosen": -491.6361389160156, + "eval_logps/rejected": -579.4501342773438, + "eval_loss": 0.49039050936698914, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -2.596311092376709, + "eval_rewards/margins": 1.0820716619491577, + "eval_rewards/rejected": -3.6783831119537354, + "eval_runtime": 732.3714, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.365, + "step": 9700 + }, + { + "epoch": 0.64, + "learning_rate": 1.7666594274699037e-06, + "logits/chosen": -0.615153968334198, + "logits/rejected": -0.585983395576477, + "logps/chosen": -514.6607055664062, + "logps/rejected": -614.2363891601562, + "loss": 0.4235, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.619704246520996, + "rewards/margins": 1.3777984380722046, + "rewards/rejected": -3.997502565383911, + "step": 9710 + }, + { + "epoch": 0.64, + "learning_rate": 1.76120261736316e-06, + "logits/chosen": -0.5721158981323242, + "logits/rejected": -0.30697041749954224, + "logps/chosen": -505.2025451660156, + "logps/rejected": -604.8211059570312, + "loss": 0.4677, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.784874677658081, + "rewards/margins": 1.2676182985305786, + "rewards/rejected": -4.052493095397949, + "step": 9720 + }, + { + "epoch": 0.64, + "learning_rate": 1.755749661185468e-06, + "logits/chosen": -0.7573305368423462, + "logits/rejected": -0.6128786206245422, + "logps/chosen": -555.46484375, + "logps/rejected": -618.6527099609375, + "loss": 0.4732, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.6101527214050293, + "rewards/margins": 1.2202155590057373, + "rewards/rejected": -3.8303680419921875, + "step": 9730 + }, + { + "epoch": 0.64, + "learning_rate": 1.7503005873821183e-06, + "logits/chosen": -0.5666888952255249, + "logits/rejected": -0.7371236085891724, + "logps/chosen": -404.9433288574219, + "logps/rejected": -559.25537109375, + "loss": 0.4454, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.4451496601104736, + "rewards/margins": 1.2804811000823975, + "rewards/rejected": -3.725630521774292, + "step": 9740 + }, + { + "epoch": 0.64, + "learning_rate": 1.744855424378148e-06, + "logits/chosen": -0.35799577832221985, + "logits/rejected": -0.7274848818778992, + "logps/chosen": -427.401611328125, + "logps/rejected": -593.9816284179688, + "loss": 0.3927, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.387746572494507, + "rewards/margins": 1.4313452243804932, + "rewards/rejected": -3.819091796875, + "step": 9750 + }, + { + "epoch": 0.64, + "learning_rate": 1.7394142005781973e-06, + "logits/chosen": -0.6731287837028503, + "logits/rejected": -0.5532606244087219, + "logps/chosen": -559.6663818359375, + "logps/rejected": -683.4664306640625, + "loss": 0.4492, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.979553699493408, + "rewards/margins": 1.2022900581359863, + "rewards/rejected": -4.1818437576293945, + "step": 9760 + }, + { + "epoch": 0.64, + "learning_rate": 1.7339769443663528e-06, + "logits/chosen": -0.5920579433441162, + "logits/rejected": -0.6127692461013794, + "logps/chosen": -395.1576232910156, + "logps/rejected": -486.21258544921875, + "loss": 0.5335, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.5221543312072754, + "rewards/margins": 0.9035172462463379, + "rewards/rejected": -3.4256718158721924, + "step": 9770 + }, + { + "epoch": 0.64, + "learning_rate": 1.7285436841060078e-06, + "logits/chosen": -0.710267186164856, + "logits/rejected": -0.5984159708023071, + "logps/chosen": -511.69659423828125, + "logps/rejected": -608.3006591796875, + "loss": 0.4099, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.355681896209717, + "rewards/margins": 1.3955835103988647, + "rewards/rejected": -3.751265287399292, + "step": 9780 + }, + { + "epoch": 0.64, + "learning_rate": 1.7231144481397083e-06, + "logits/chosen": -0.8003692626953125, + "logits/rejected": -0.676064670085907, + "logps/chosen": -445.18719482421875, + "logps/rejected": -521.8858642578125, + "loss": 0.5414, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.2478883266448975, + "rewards/margins": 1.0213735103607178, + "rewards/rejected": -3.2692618370056152, + "step": 9790 + }, + { + "epoch": 0.64, + "learning_rate": 1.7176892647890092e-06, + "logits/chosen": -0.6346590518951416, + "logits/rejected": -0.31158536672592163, + "logps/chosen": -501.07159423828125, + "logps/rejected": -548.5491943359375, + "loss": 0.5332, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.6540558338165283, + "rewards/margins": 0.8635900616645813, + "rewards/rejected": -3.517646074295044, + "step": 9800 + }, + { + "epoch": 0.64, + "eval_logits/chosen": -0.6229720115661621, + "eval_logits/rejected": -0.5179995894432068, + "eval_logps/chosen": -486.57135009765625, + "eval_logps/rejected": -579.4781494140625, + "eval_loss": 0.49061888456344604, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -2.545663595199585, + "eval_rewards/margins": 1.132997751235962, + "eval_rewards/rejected": -3.678661584854126, + "eval_runtime": 735.253, + "eval_samples_per_second": 2.72, + "eval_steps_per_second": 1.36, + "step": 9800 + }, + { + "epoch": 0.64, + "learning_rate": 1.7122681623543239e-06, + "logits/chosen": -0.755142092704773, + "logits/rejected": -0.8409514427185059, + "logps/chosen": -506.37664794921875, + "logps/rejected": -651.1091918945312, + "loss": 0.3644, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.5938804149627686, + "rewards/margins": 1.5594637393951416, + "rewards/rejected": -4.15334415435791, + "step": 9810 + }, + { + "epoch": 0.64, + "learning_rate": 1.7068511691147788e-06, + "logits/chosen": -0.5418431758880615, + "logits/rejected": -0.46902474761009216, + "logps/chosen": -425.26593017578125, + "logps/rejected": -558.49365234375, + "loss": 0.4844, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3113489151000977, + "rewards/margins": 1.2485809326171875, + "rewards/rejected": -3.559929609298706, + "step": 9820 + }, + { + "epoch": 0.64, + "learning_rate": 1.7014383133280636e-06, + "logits/chosen": -0.7209473848342896, + "logits/rejected": -0.3792896866798401, + "logps/chosen": -532.0540771484375, + "logps/rejected": -575.2877197265625, + "loss": 0.5814, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.834196090698242, + "rewards/margins": 0.9662758708000183, + "rewards/rejected": -3.8004722595214844, + "step": 9830 + }, + { + "epoch": 0.64, + "learning_rate": 1.696029623230286e-06, + "logits/chosen": -0.6430082321166992, + "logits/rejected": -0.7621620893478394, + "logps/chosen": -549.1597900390625, + "logps/rejected": -683.6041870117188, + "loss": 0.5399, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.9323744773864746, + "rewards/margins": 1.156652808189392, + "rewards/rejected": -4.089027404785156, + "step": 9840 + }, + { + "epoch": 0.64, + "learning_rate": 1.6906251270358229e-06, + "logits/chosen": -0.8286713361740112, + "logits/rejected": -0.7480670213699341, + "logps/chosen": -508.75213623046875, + "logps/rejected": -546.1533813476562, + "loss": 0.4749, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.4589786529541016, + "rewards/margins": 0.8714302182197571, + "rewards/rejected": -3.3304085731506348, + "step": 9850 + }, + { + "epoch": 0.65, + "learning_rate": 1.685224852937174e-06, + "logits/chosen": -0.6186366677284241, + "logits/rejected": -0.2713349461555481, + "logps/chosen": -400.1719055175781, + "logps/rejected": -645.1832275390625, + "loss": 0.3151, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.018071174621582, + "rewards/margins": 2.0315232276916504, + "rewards/rejected": -4.049594402313232, + "step": 9860 + }, + { + "epoch": 0.65, + "learning_rate": 1.6798288291048136e-06, + "logits/chosen": -0.5639387369155884, + "logits/rejected": -0.5140062570571899, + "logps/chosen": -461.98919677734375, + "logps/rejected": -567.0389404296875, + "loss": 0.4877, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3874754905700684, + "rewards/margins": 1.3375474214553833, + "rewards/rejected": -3.725022554397583, + "step": 9870 + }, + { + "epoch": 0.65, + "learning_rate": 1.6744370836870466e-06, + "logits/chosen": -1.273271083831787, + "logits/rejected": -0.8345822095870972, + "logps/chosen": -565.0169067382812, + "logps/rejected": -594.915771484375, + "loss": 0.6041, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.3032336235046387, + "rewards/margins": 1.1341663599014282, + "rewards/rejected": -3.4374001026153564, + "step": 9880 + }, + { + "epoch": 0.65, + "learning_rate": 1.6690496448098576e-06, + "logits/chosen": -0.612956166267395, + "logits/rejected": -0.46901369094848633, + "logps/chosen": -441.619873046875, + "logps/rejected": -527.1732177734375, + "loss": 0.517, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.125836133956909, + "rewards/margins": 1.0641244649887085, + "rewards/rejected": -3.1899607181549072, + "step": 9890 + }, + { + "epoch": 0.65, + "learning_rate": 1.6636665405767666e-06, + "logits/chosen": -0.5108221769332886, + "logits/rejected": -0.4370543360710144, + "logps/chosen": -450.4356384277344, + "logps/rejected": -529.3080444335938, + "loss": 0.524, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1266462802886963, + "rewards/margins": 0.977624773979187, + "rewards/rejected": -3.104271173477173, + "step": 9900 + }, + { + "epoch": 0.65, + "eval_logits/chosen": -0.7448288202285767, + "eval_logits/rejected": -0.6354550719261169, + "eval_logps/chosen": -445.274169921875, + "eval_logps/rejected": -526.677001953125, + "eval_loss": 0.4900985360145569, + "eval_rewards/accuracies": 0.675000011920929, + "eval_rewards/chosen": -2.1326918601989746, + "eval_rewards/margins": 1.0179589986801147, + "eval_rewards/rejected": -3.150651216506958, + "eval_runtime": 734.6282, + "eval_samples_per_second": 2.722, + "eval_steps_per_second": 1.361, + "step": 9900 + }, + { + "epoch": 0.65, + "learning_rate": 1.6582877990686827e-06, + "logits/chosen": -0.7203329801559448, + "logits/rejected": -0.8921548128128052, + "logps/chosen": -297.42352294921875, + "logps/rejected": -421.01983642578125, + "loss": 0.4527, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.7612918615341187, + "rewards/margins": 1.012649655342102, + "rewards/rejected": -2.7739417552948, + "step": 9910 + }, + { + "epoch": 0.65, + "learning_rate": 1.6529134483437562e-06, + "logits/chosen": -0.6714494228363037, + "logits/rejected": -0.8752210736274719, + "logps/chosen": -390.29119873046875, + "logps/rejected": -467.7923889160156, + "loss": 0.3939, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.847169280052185, + "rewards/margins": 1.1695491075515747, + "rewards/rejected": -3.0167183876037598, + "step": 9920 + }, + { + "epoch": 0.65, + "learning_rate": 1.647543516437233e-06, + "logits/chosen": -0.9732862710952759, + "logits/rejected": -0.9849495887756348, + "logps/chosen": -392.1590881347656, + "logps/rejected": -500.3994140625, + "loss": 0.5757, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.931208610534668, + "rewards/margins": 0.8755546808242798, + "rewards/rejected": -2.8067634105682373, + "step": 9930 + }, + { + "epoch": 0.65, + "learning_rate": 1.6421780313613088e-06, + "logits/chosen": -0.7765932083129883, + "logits/rejected": -0.4009782671928406, + "logps/chosen": -403.697509765625, + "logps/rejected": -510.45379638671875, + "loss": 0.4019, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.9895591735839844, + "rewards/margins": 1.3994615077972412, + "rewards/rejected": -3.3890204429626465, + "step": 9940 + }, + { + "epoch": 0.65, + "learning_rate": 1.6368170211049816e-06, + "logits/chosen": -0.47321072220802307, + "logits/rejected": -0.4680631756782532, + "logps/chosen": -521.65673828125, + "logps/rejected": -566.8966674804688, + "loss": 0.583, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.378565549850464, + "rewards/margins": 1.0535138845443726, + "rewards/rejected": -3.432079792022705, + "step": 9950 + }, + { + "epoch": 0.65, + "learning_rate": 1.6314605136339074e-06, + "logits/chosen": -0.8041669726371765, + "logits/rejected": -0.6485110521316528, + "logps/chosen": -404.8415832519531, + "logps/rejected": -462.87188720703125, + "loss": 0.5907, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.0925488471984863, + "rewards/margins": 0.7592976689338684, + "rewards/rejected": -2.851846218109131, + "step": 9960 + }, + { + "epoch": 0.65, + "learning_rate": 1.6261085368902526e-06, + "logits/chosen": -1.1480224132537842, + "logits/rejected": -0.9943176507949829, + "logps/chosen": -510.9529724121094, + "logps/rejected": -532.6697998046875, + "loss": 0.5558, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.460120677947998, + "rewards/margins": 0.6534627676010132, + "rewards/rejected": -3.11358380317688, + "step": 9970 + }, + { + "epoch": 0.65, + "learning_rate": 1.6207611187925503e-06, + "logits/chosen": -0.9005382657051086, + "logits/rejected": -0.838882327079773, + "logps/chosen": -399.6454772949219, + "logps/rejected": -568.9725952148438, + "loss": 0.3687, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.9081859588623047, + "rewards/margins": 1.230093002319336, + "rewards/rejected": -3.1382789611816406, + "step": 9980 + }, + { + "epoch": 0.65, + "learning_rate": 1.6154182872355512e-06, + "logits/chosen": -0.7568344473838806, + "logits/rejected": -0.9104105234146118, + "logps/chosen": -360.6561584472656, + "logps/rejected": -448.56671142578125, + "loss": 0.6373, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9683866500854492, + "rewards/margins": 0.7349096536636353, + "rewards/rejected": -2.703296422958374, + "step": 9990 + }, + { + "epoch": 0.65, + "learning_rate": 1.610080070090084e-06, + "logits/chosen": -0.9353153109550476, + "logits/rejected": -0.8025709390640259, + "logps/chosen": -384.53924560546875, + "logps/rejected": -476.81915283203125, + "loss": 0.4316, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.036179304122925, + "rewards/margins": 1.0130820274353027, + "rewards/rejected": -3.0492615699768066, + "step": 10000 + }, + { + "epoch": 0.65, + "eval_logits/chosen": -0.8593093156814575, + "eval_logits/rejected": -0.743216872215271, + "eval_logps/chosen": -431.44866943359375, + "eval_logps/rejected": -515.6310424804688, + "eval_loss": 0.4896312355995178, + "eval_rewards/accuracies": 0.6725000143051147, + "eval_rewards/chosen": -1.9944367408752441, + "eval_rewards/margins": 1.0457544326782227, + "eval_rewards/rejected": -3.0401909351348877, + "eval_runtime": 734.4039, + "eval_samples_per_second": 2.723, + "eval_steps_per_second": 1.362, + "step": 10000 + }, + { + "epoch": 0.65, + "learning_rate": 1.6047464952029034e-06, + "logits/chosen": -1.1312496662139893, + "logits/rejected": -1.077750563621521, + "logps/chosen": -442.98260498046875, + "logps/rejected": -593.0875244140625, + "loss": 0.3385, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8869520425796509, + "rewards/margins": 1.4187886714935303, + "rewards/rejected": -3.3057403564453125, + "step": 10010 + }, + { + "epoch": 0.66, + "learning_rate": 1.5994175903965486e-06, + "logits/chosen": -0.7303309440612793, + "logits/rejected": -0.5155562162399292, + "logps/chosen": -452.1363830566406, + "logps/rejected": -560.2066650390625, + "loss": 0.4909, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0180599689483643, + "rewards/margins": 1.0631579160690308, + "rewards/rejected": -3.0812180042266846, + "step": 10020 + }, + { + "epoch": 0.66, + "learning_rate": 1.5940933834691977e-06, + "logits/chosen": -1.207631230354309, + "logits/rejected": -0.8119298219680786, + "logps/chosen": -488.1917419433594, + "logps/rejected": -507.716064453125, + "loss": 0.4227, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8292583227157593, + "rewards/margins": 1.2742161750793457, + "rewards/rejected": -3.1034743785858154, + "step": 10030 + }, + { + "epoch": 0.66, + "learning_rate": 1.588773902194522e-06, + "logits/chosen": -0.8028233647346497, + "logits/rejected": -0.5262702107429504, + "logps/chosen": -406.1075439453125, + "logps/rejected": -532.1007080078125, + "loss": 0.4059, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.0202784538269043, + "rewards/margins": 1.3023350238800049, + "rewards/rejected": -3.322613477706909, + "step": 10040 + }, + { + "epoch": 0.66, + "learning_rate": 1.583459174321541e-06, + "logits/chosen": -0.614817202091217, + "logits/rejected": -0.6362075805664062, + "logps/chosen": -402.9064636230469, + "logps/rejected": -498.8567810058594, + "loss": 0.4797, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.932891607284546, + "rewards/margins": 1.2304433584213257, + "rewards/rejected": -3.163335084915161, + "step": 10050 + }, + { + "epoch": 0.66, + "learning_rate": 1.5781492275744797e-06, + "logits/chosen": -1.2567778825759888, + "logits/rejected": -1.0947939157485962, + "logps/chosen": -487.99322509765625, + "logps/rejected": -631.7654418945312, + "loss": 0.412, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.9374876022338867, + "rewards/margins": 1.6999012231826782, + "rewards/rejected": -3.6373889446258545, + "step": 10060 + }, + { + "epoch": 0.66, + "learning_rate": 1.5728440896526215e-06, + "logits/chosen": -0.7466986775398254, + "logits/rejected": -0.670479953289032, + "logps/chosen": -459.1402282714844, + "logps/rejected": -534.97216796875, + "loss": 0.4596, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8573827743530273, + "rewards/margins": 1.2203471660614014, + "rewards/rejected": -3.077730178833008, + "step": 10070 + }, + { + "epoch": 0.66, + "learning_rate": 1.5675437882301633e-06, + "logits/chosen": -0.9598585367202759, + "logits/rejected": -0.9147456884384155, + "logps/chosen": -407.67547607421875, + "logps/rejected": -435.04742431640625, + "loss": 0.5896, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.8863128423690796, + "rewards/margins": 0.6216771006584167, + "rewards/rejected": -2.5079903602600098, + "step": 10080 + }, + { + "epoch": 0.66, + "learning_rate": 1.5622483509560748e-06, + "logits/chosen": -0.8460068702697754, + "logits/rejected": -0.9365545511245728, + "logps/chosen": -343.73724365234375, + "logps/rejected": -474.4576721191406, + "loss": 0.5243, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7300806045532227, + "rewards/margins": 1.0053818225860596, + "rewards/rejected": -2.7354624271392822, + "step": 10090 + }, + { + "epoch": 0.66, + "learning_rate": 1.5569578054539506e-06, + "logits/chosen": -1.0083024501800537, + "logits/rejected": -0.6957250833511353, + "logps/chosen": -470.3836364746094, + "logps/rejected": -570.5111083984375, + "loss": 0.3164, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9118655920028687, + "rewards/margins": 1.8399053812026978, + "rewards/rejected": -3.7517712116241455, + "step": 10100 + }, + { + "epoch": 0.66, + "eval_logits/chosen": -0.9509793519973755, + "eval_logits/rejected": -0.8275907635688782, + "eval_logps/chosen": -418.5705261230469, + "eval_logps/rejected": -511.33795166015625, + "eval_loss": 0.4899994134902954, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -1.865655541419983, + "eval_rewards/margins": 1.131604790687561, + "eval_rewards/rejected": -2.997260570526123, + "eval_runtime": 732.5384, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 10100 + }, + { + "epoch": 0.66, + "learning_rate": 1.551672179321867e-06, + "logits/chosen": -0.9406857490539551, + "logits/rejected": -0.9383149147033691, + "logps/chosen": -386.16998291015625, + "logps/rejected": -473.0233459472656, + "loss": 0.4964, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7031307220458984, + "rewards/margins": 1.080244541168213, + "rewards/rejected": -2.7833752632141113, + "step": 10110 + }, + { + "epoch": 0.66, + "learning_rate": 1.5463915001322398e-06, + "logits/chosen": -0.9525520205497742, + "logits/rejected": -0.776652455329895, + "logps/chosen": -439.74163818359375, + "logps/rejected": -530.9915161132812, + "loss": 0.639, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9408528804779053, + "rewards/margins": 1.0561749935150146, + "rewards/rejected": -2.997027635574341, + "step": 10120 + }, + { + "epoch": 0.66, + "learning_rate": 1.5411157954316784e-06, + "logits/chosen": -1.1934840679168701, + "logits/rejected": -0.8447572588920593, + "logps/chosen": -347.8656005859375, + "logps/rejected": -438.13330078125, + "loss": 0.4629, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.4598805904388428, + "rewards/margins": 1.0395748615264893, + "rewards/rejected": -2.499455690383911, + "step": 10130 + }, + { + "epoch": 0.66, + "learning_rate": 1.535845092740843e-06, + "logits/chosen": -1.065664529800415, + "logits/rejected": -1.037608027458191, + "logps/chosen": -403.48992919921875, + "logps/rejected": -487.27313232421875, + "loss": 0.5483, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.6616871356964111, + "rewards/margins": 0.7138743996620178, + "rewards/rejected": -2.375561475753784, + "step": 10140 + }, + { + "epoch": 0.66, + "learning_rate": 1.5305794195543005e-06, + "logits/chosen": -1.1213363409042358, + "logits/rejected": -1.1167705059051514, + "logps/chosen": -382.3546447753906, + "logps/rejected": -487.45135498046875, + "loss": 0.4169, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.7517554759979248, + "rewards/margins": 1.2202813625335693, + "rewards/rejected": -2.9720370769500732, + "step": 10150 + }, + { + "epoch": 0.66, + "learning_rate": 1.5253188033403816e-06, + "logits/chosen": -1.1482348442077637, + "logits/rejected": -1.0655076503753662, + "logps/chosen": -344.8389892578125, + "logps/rejected": -426.26483154296875, + "loss": 0.5629, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.7410409450531006, + "rewards/margins": 0.6476749181747437, + "rewards/rejected": -2.388715982437134, + "step": 10160 + }, + { + "epoch": 0.67, + "learning_rate": 1.520063271541037e-06, + "logits/chosen": -0.9673888087272644, + "logits/rejected": -0.8863734006881714, + "logps/chosen": -423.4639587402344, + "logps/rejected": -525.8157958984375, + "loss": 0.5195, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.449592113494873, + "rewards/margins": 1.1365474462509155, + "rewards/rejected": -3.5861401557922363, + "step": 10170 + }, + { + "epoch": 0.67, + "learning_rate": 1.5148128515716954e-06, + "logits/chosen": -1.0516979694366455, + "logits/rejected": -0.6746267080307007, + "logps/chosen": -480.287109375, + "logps/rejected": -529.2515869140625, + "loss": 0.489, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.1375792026519775, + "rewards/margins": 1.2133866548538208, + "rewards/rejected": -3.350966215133667, + "step": 10180 + }, + { + "epoch": 0.67, + "learning_rate": 1.5095675708211197e-06, + "logits/chosen": -0.9321045875549316, + "logits/rejected": -0.8480215072631836, + "logps/chosen": -450.95098876953125, + "logps/rejected": -511.4710388183594, + "loss": 0.6915, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -2.597900867462158, + "rewards/margins": 0.37176448106765747, + "rewards/rejected": -2.96966552734375, + "step": 10190 + }, + { + "epoch": 0.67, + "learning_rate": 1.504327456651263e-06, + "logits/chosen": -0.6907519698143005, + "logits/rejected": -0.5835791826248169, + "logps/chosen": -530.9114379882812, + "logps/rejected": -577.1619873046875, + "loss": 0.517, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.695084810256958, + "rewards/margins": 0.7321516275405884, + "rewards/rejected": -3.427236557006836, + "step": 10200 + }, + { + "epoch": 0.67, + "eval_logits/chosen": -0.8518573641777039, + "eval_logits/rejected": -0.7372215390205383, + "eval_logps/chosen": -465.50921630859375, + "eval_logps/rejected": -543.9869995117188, + "eval_loss": 0.492562472820282, + "eval_rewards/accuracies": 0.6679999828338623, + "eval_rewards/chosen": -2.335042715072632, + "eval_rewards/margins": 0.9887086153030396, + "eval_rewards/rejected": -3.323751211166382, + "eval_runtime": 733.657, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 10200 + }, + { + "epoch": 0.67, + "learning_rate": 1.4990925363971284e-06, + "logits/chosen": -0.9815598726272583, + "logits/rejected": -0.504173755645752, + "logps/chosen": -545.0457763671875, + "logps/rejected": -634.4996337890625, + "loss": 0.3686, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.5512261390686035, + "rewards/margins": 1.5794289112091064, + "rewards/rejected": -4.130654811859131, + "step": 10210 + }, + { + "epoch": 0.67, + "learning_rate": 1.4938628373666236e-06, + "logits/chosen": -0.8545439839363098, + "logits/rejected": -0.768897533416748, + "logps/chosen": -392.7917175292969, + "logps/rejected": -484.1936950683594, + "loss": 0.5518, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -2.202124834060669, + "rewards/margins": 0.8952932357788086, + "rewards/rejected": -3.0974178314208984, + "step": 10220 + }, + { + "epoch": 0.67, + "learning_rate": 1.4886383868404203e-06, + "logits/chosen": -0.6459547281265259, + "logits/rejected": -0.7341974973678589, + "logps/chosen": -368.89923095703125, + "logps/rejected": -473.0926208496094, + "loss": 0.4036, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.076774835586548, + "rewards/margins": 1.1422326564788818, + "rewards/rejected": -3.2190074920654297, + "step": 10230 + }, + { + "epoch": 0.67, + "learning_rate": 1.483419212071813e-06, + "logits/chosen": -0.5583785176277161, + "logits/rejected": -0.3464600741863251, + "logps/chosen": -415.6678771972656, + "logps/rejected": -493.46759033203125, + "loss": 0.5239, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2720372676849365, + "rewards/margins": 0.8498104810714722, + "rewards/rejected": -3.1218478679656982, + "step": 10240 + }, + { + "epoch": 0.67, + "learning_rate": 1.478205340286573e-06, + "logits/chosen": -0.8108803033828735, + "logits/rejected": -0.9261777997016907, + "logps/chosen": -460.70050048828125, + "logps/rejected": -527.3087158203125, + "loss": 0.56, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.570915937423706, + "rewards/margins": 0.8461298942565918, + "rewards/rejected": -3.4170455932617188, + "step": 10250 + }, + { + "epoch": 0.67, + "learning_rate": 1.4729967986828104e-06, + "logits/chosen": -0.8202739953994751, + "logits/rejected": -0.7946422696113586, + "logps/chosen": -569.1189575195312, + "logps/rejected": -645.8231201171875, + "loss": 0.4275, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.4713521003723145, + "rewards/margins": 1.2467219829559326, + "rewards/rejected": -3.718074083328247, + "step": 10260 + }, + { + "epoch": 0.67, + "learning_rate": 1.4677936144308286e-06, + "logits/chosen": -0.8981763124465942, + "logits/rejected": -0.6755679249763489, + "logps/chosen": -467.2249450683594, + "logps/rejected": -572.6213989257812, + "loss": 0.4523, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.4298019409179688, + "rewards/margins": 1.3345946073532104, + "rewards/rejected": -3.7643961906433105, + "step": 10270 + }, + { + "epoch": 0.67, + "learning_rate": 1.4625958146729864e-06, + "logits/chosen": -1.0698392391204834, + "logits/rejected": -0.7566840052604675, + "logps/chosen": -467.49822998046875, + "logps/rejected": -540.9188232421875, + "loss": 0.5114, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.472343921661377, + "rewards/margins": 0.8936858177185059, + "rewards/rejected": -3.366029739379883, + "step": 10280 + }, + { + "epoch": 0.67, + "learning_rate": 1.4574034265235523e-06, + "logits/chosen": -0.799394428730011, + "logits/rejected": -0.6376802325248718, + "logps/chosen": -481.4248046875, + "logps/rejected": -503.7127990722656, + "loss": 0.4684, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.255962371826172, + "rewards/margins": 1.2005085945129395, + "rewards/rejected": -3.4564712047576904, + "step": 10290 + }, + { + "epoch": 0.67, + "learning_rate": 1.452216477068568e-06, + "logits/chosen": -0.6714291572570801, + "logits/rejected": -0.465345561504364, + "logps/chosen": -427.8998107910156, + "logps/rejected": -462.84124755859375, + "loss": 0.4479, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9785480499267578, + "rewards/margins": 1.278562307357788, + "rewards/rejected": -3.257110118865967, + "step": 10300 + }, + { + "epoch": 0.67, + "eval_logits/chosen": -0.8858751058578491, + "eval_logits/rejected": -0.7681257128715515, + "eval_logps/chosen": -471.5842590332031, + "eval_logps/rejected": -554.7045288085938, + "eval_loss": 0.4910919666290283, + "eval_rewards/accuracies": 0.6639999747276306, + "eval_rewards/chosen": -2.3957931995391846, + "eval_rewards/margins": 1.0351324081420898, + "eval_rewards/rejected": -3.4309258460998535, + "eval_runtime": 732.4777, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 10300 + }, + { + "epoch": 0.67, + "learning_rate": 1.4470349933657004e-06, + "logits/chosen": -1.418860673904419, + "logits/rejected": -0.9940776824951172, + "logps/chosen": -454.7064514160156, + "logps/rejected": -535.0302124023438, + "loss": 0.4524, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.339768171310425, + "rewards/margins": 1.0430269241333008, + "rewards/rejected": -3.3827946186065674, + "step": 10310 + }, + { + "epoch": 0.68, + "learning_rate": 1.4418590024441096e-06, + "logits/chosen": -1.18935227394104, + "logits/rejected": -0.697831928730011, + "logps/chosen": -467.32855224609375, + "logps/rejected": -498.80645751953125, + "loss": 0.5162, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1386830806732178, + "rewards/margins": 0.9796315431594849, + "rewards/rejected": -3.118314266204834, + "step": 10320 + }, + { + "epoch": 0.68, + "learning_rate": 1.436688531304297e-06, + "logits/chosen": -1.0277507305145264, + "logits/rejected": -0.8398463129997253, + "logps/chosen": -437.97479248046875, + "logps/rejected": -539.4046630859375, + "loss": 0.5487, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.196472644805908, + "rewards/margins": 1.0691899061203003, + "rewards/rejected": -3.265662670135498, + "step": 10330 + }, + { + "epoch": 0.68, + "learning_rate": 1.431523606917974e-06, + "logits/chosen": -0.9352237582206726, + "logits/rejected": -0.8883055448532104, + "logps/chosen": -485.72412109375, + "logps/rejected": -579.2689208984375, + "loss": 0.6787, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.806248188018799, + "rewards/margins": 0.8693000078201294, + "rewards/rejected": -3.675548553466797, + "step": 10340 + }, + { + "epoch": 0.68, + "learning_rate": 1.4263642562279162e-06, + "logits/chosen": -0.6453313827514648, + "logits/rejected": -0.5077480673789978, + "logps/chosen": -519.162353515625, + "logps/rejected": -649.4414672851562, + "loss": 0.431, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.6967499256134033, + "rewards/margins": 1.2178905010223389, + "rewards/rejected": -3.9146409034729004, + "step": 10350 + }, + { + "epoch": 0.68, + "learning_rate": 1.4212105061478257e-06, + "logits/chosen": -0.7289652228355408, + "logits/rejected": -0.46540945768356323, + "logps/chosen": -536.9349365234375, + "logps/rejected": -655.5059814453125, + "loss": 0.5523, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -3.070444345474243, + "rewards/margins": 1.1074758768081665, + "rewards/rejected": -4.177920341491699, + "step": 10360 + }, + { + "epoch": 0.68, + "learning_rate": 1.4160623835621848e-06, + "logits/chosen": -1.2176058292388916, + "logits/rejected": -0.8479671478271484, + "logps/chosen": -469.68682861328125, + "logps/rejected": -566.1956787109375, + "loss": 0.4645, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.323755979537964, + "rewards/margins": 1.0798704624176025, + "rewards/rejected": -3.4036262035369873, + "step": 10370 + }, + { + "epoch": 0.68, + "learning_rate": 1.4109199153261249e-06, + "logits/chosen": -0.9392701387405396, + "logits/rejected": -0.7439551949501038, + "logps/chosen": -543.1309204101562, + "logps/rejected": -624.5114135742188, + "loss": 0.4741, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.67136287689209, + "rewards/margins": 1.1169406175613403, + "rewards/rejected": -3.7883033752441406, + "step": 10380 + }, + { + "epoch": 0.68, + "learning_rate": 1.405783128265278e-06, + "logits/chosen": -1.0329110622406006, + "logits/rejected": -0.9256108999252319, + "logps/chosen": -479.529541015625, + "logps/rejected": -570.4544677734375, + "loss": 0.5458, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.753960371017456, + "rewards/margins": 0.9419110417366028, + "rewards/rejected": -3.695871353149414, + "step": 10390 + }, + { + "epoch": 0.68, + "learning_rate": 1.4006520491756427e-06, + "logits/chosen": -0.9245737791061401, + "logits/rejected": -0.6148309707641602, + "logps/chosen": -415.374755859375, + "logps/rejected": -461.10223388671875, + "loss": 0.4663, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.191504955291748, + "rewards/margins": 1.1036841869354248, + "rewards/rejected": -3.295189619064331, + "step": 10400 + }, + { + "epoch": 0.68, + "eval_logits/chosen": -0.9938727021217346, + "eval_logits/rejected": -0.8684021234512329, + "eval_logps/chosen": -437.4019470214844, + "eval_logps/rejected": -522.1436157226562, + "eval_loss": 0.4915103614330292, + "eval_rewards/accuracies": 0.6675000190734863, + "eval_rewards/chosen": -2.0539698600769043, + "eval_rewards/margins": 1.0513461828231812, + "eval_rewards/rejected": -3.105316400527954, + "eval_runtime": 733.2105, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 10400 + }, + { + "epoch": 0.68, + "learning_rate": 1.39552670482344e-06, + "logits/chosen": -0.9635096788406372, + "logits/rejected": -1.0806286334991455, + "logps/chosen": -363.7113952636719, + "logps/rejected": -444.799072265625, + "loss": 0.5617, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.891117811203003, + "rewards/margins": 0.8119624853134155, + "rewards/rejected": -2.703080177307129, + "step": 10410 + }, + { + "epoch": 0.68, + "learning_rate": 1.3904071219449776e-06, + "logits/chosen": -0.9084879755973816, + "logits/rejected": -0.6438118815422058, + "logps/chosen": -374.66009521484375, + "logps/rejected": -393.7281799316406, + "loss": 0.482, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7475841045379639, + "rewards/margins": 1.0372825860977173, + "rewards/rejected": -2.7848668098449707, + "step": 10420 + }, + { + "epoch": 0.68, + "learning_rate": 1.3852933272465068e-06, + "logits/chosen": -1.0169692039489746, + "logits/rejected": -0.9037554860115051, + "logps/chosen": -393.3205871582031, + "logps/rejected": -454.0645446777344, + "loss": 0.4999, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.5546047687530518, + "rewards/margins": 0.9596779942512512, + "rewards/rejected": -2.514282703399658, + "step": 10430 + }, + { + "epoch": 0.68, + "learning_rate": 1.3801853474040873e-06, + "logits/chosen": -0.9115139245986938, + "logits/rejected": -0.8385306596755981, + "logps/chosen": -459.4090270996094, + "logps/rejected": -559.7108154296875, + "loss": 0.4073, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.2088780403137207, + "rewards/margins": 1.1676206588745117, + "rewards/rejected": -3.3764984607696533, + "step": 10440 + }, + { + "epoch": 0.68, + "learning_rate": 1.3750832090634417e-06, + "logits/chosen": -1.0661998987197876, + "logits/rejected": -0.8567445874214172, + "logps/chosen": -379.2359313964844, + "logps/rejected": -462.881103515625, + "loss": 0.5572, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9306328296661377, + "rewards/margins": 0.9829649925231934, + "rewards/rejected": -2.913597822189331, + "step": 10450 + }, + { + "epoch": 0.68, + "learning_rate": 1.3699869388398245e-06, + "logits/chosen": -0.892473042011261, + "logits/rejected": -0.7880361676216125, + "logps/chosen": -453.9684143066406, + "logps/rejected": -541.6419677734375, + "loss": 0.5119, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.4094717502593994, + "rewards/margins": 1.0612695217132568, + "rewards/rejected": -3.470740795135498, + "step": 10460 + }, + { + "epoch": 0.69, + "learning_rate": 1.3648965633178772e-06, + "logits/chosen": -1.0346626043319702, + "logits/rejected": -0.9228593707084656, + "logps/chosen": -419.2098693847656, + "logps/rejected": -561.0824584960938, + "loss": 0.4033, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.184777021408081, + "rewards/margins": 1.304609775543213, + "rewards/rejected": -3.489386796951294, + "step": 10470 + }, + { + "epoch": 0.69, + "learning_rate": 1.3598121090514938e-06, + "logits/chosen": -0.790840744972229, + "logits/rejected": -0.8971914052963257, + "logps/chosen": -388.94677734375, + "logps/rejected": -464.78369140625, + "loss": 0.499, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.059389591217041, + "rewards/margins": 0.9996012449264526, + "rewards/rejected": -3.058990955352783, + "step": 10480 + }, + { + "epoch": 0.69, + "learning_rate": 1.3547336025636753e-06, + "logits/chosen": -0.8925707936286926, + "logits/rejected": -0.6376835703849792, + "logps/chosen": -498.9082946777344, + "logps/rejected": -568.8594970703125, + "loss": 0.4828, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.2361178398132324, + "rewards/margins": 1.1148022413253784, + "rewards/rejected": -3.350919723510742, + "step": 10490 + }, + { + "epoch": 0.69, + "learning_rate": 1.3496610703464022e-06, + "logits/chosen": -1.1685224771499634, + "logits/rejected": -0.8341997265815735, + "logps/chosen": -453.0849609375, + "logps/rejected": -490.42840576171875, + "loss": 0.5752, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.238703727722168, + "rewards/margins": 0.8286484479904175, + "rewards/rejected": -3.067352771759033, + "step": 10500 + }, + { + "epoch": 0.69, + "eval_logits/chosen": -1.0515855550765991, + "eval_logits/rejected": -0.9209389090538025, + "eval_logps/chosen": -436.2606506347656, + "eval_logps/rejected": -528.1688842773438, + "eval_loss": 0.4914741814136505, + "eval_rewards/accuracies": 0.6679999828338623, + "eval_rewards/chosen": -2.04255747795105, + "eval_rewards/margins": 1.1230124235153198, + "eval_rewards/rejected": -3.165569543838501, + "eval_runtime": 733.7382, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 10500 + }, + { + "epoch": 0.69, + "learning_rate": 1.3445945388604848e-06, + "logits/chosen": -1.14806067943573, + "logits/rejected": -0.693806529045105, + "logps/chosen": -479.4612731933594, + "logps/rejected": -563.4382934570312, + "loss": 0.5061, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.4123778343200684, + "rewards/margins": 1.2713767290115356, + "rewards/rejected": -3.6837546825408936, + "step": 10510 + }, + { + "epoch": 0.69, + "learning_rate": 1.3395340345354358e-06, + "logits/chosen": -1.1252193450927734, + "logits/rejected": -1.1935184001922607, + "logps/chosen": -446.6983337402344, + "logps/rejected": -558.856689453125, + "loss": 0.5534, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2300052642822266, + "rewards/margins": 0.9491079449653625, + "rewards/rejected": -3.1791129112243652, + "step": 10520 + }, + { + "epoch": 0.69, + "learning_rate": 1.334479583769322e-06, + "logits/chosen": -1.2889360189437866, + "logits/rejected": -1.2451645135879517, + "logps/chosen": -458.30194091796875, + "logps/rejected": -509.0130920410156, + "loss": 0.4664, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0346317291259766, + "rewards/margins": 1.0068074464797974, + "rewards/rejected": -3.0414390563964844, + "step": 10530 + }, + { + "epoch": 0.69, + "learning_rate": 1.3294312129286366e-06, + "logits/chosen": -1.0461938381195068, + "logits/rejected": -0.9540836215019226, + "logps/chosen": -434.68817138671875, + "logps/rejected": -522.4949951171875, + "loss": 0.4361, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.6795955896377563, + "rewards/margins": 1.014776587486267, + "rewards/rejected": -2.6943724155426025, + "step": 10540 + }, + { + "epoch": 0.69, + "learning_rate": 1.324388948348153e-06, + "logits/chosen": -1.3969929218292236, + "logits/rejected": -1.0430009365081787, + "logps/chosen": -457.0474548339844, + "logps/rejected": -478.5557556152344, + "loss": 0.4524, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7180945873260498, + "rewards/margins": 1.028747797012329, + "rewards/rejected": -2.7468421459198, + "step": 10550 + }, + { + "epoch": 0.69, + "learning_rate": 1.319352816330796e-06, + "logits/chosen": -1.3804547786712646, + "logits/rejected": -0.9653580784797668, + "logps/chosen": -473.27630615234375, + "logps/rejected": -479.4024353027344, + "loss": 0.4872, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9007914066314697, + "rewards/margins": 1.1258151531219482, + "rewards/rejected": -3.026606559753418, + "step": 10560 + }, + { + "epoch": 0.69, + "learning_rate": 1.314322843147494e-06, + "logits/chosen": -0.979723334312439, + "logits/rejected": -1.067060112953186, + "logps/chosen": -402.91583251953125, + "logps/rejected": -531.9745483398438, + "loss": 0.5855, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.325900077819824, + "rewards/margins": 0.7419096231460571, + "rewards/rejected": -3.067809581756592, + "step": 10570 + }, + { + "epoch": 0.69, + "learning_rate": 1.3092990550370526e-06, + "logits/chosen": -1.077411413192749, + "logits/rejected": -0.9932012557983398, + "logps/chosen": -569.3215942382812, + "logps/rejected": -607.20263671875, + "loss": 0.5044, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2356317043304443, + "rewards/margins": 1.2323225736618042, + "rewards/rejected": -3.467954158782959, + "step": 10580 + }, + { + "epoch": 0.69, + "learning_rate": 1.3042814782060131e-06, + "logits/chosen": -0.7737795114517212, + "logits/rejected": -0.7166412472724915, + "logps/chosen": -329.2445068359375, + "logps/rejected": -439.805908203125, + "loss": 0.4284, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.4774162769317627, + "rewards/margins": 1.3447765111923218, + "rewards/rejected": -2.822192668914795, + "step": 10590 + }, + { + "epoch": 0.69, + "learning_rate": 1.2992701388285112e-06, + "logits/chosen": -0.8949775695800781, + "logits/rejected": -0.774708092212677, + "logps/chosen": -445.0848693847656, + "logps/rejected": -506.23480224609375, + "loss": 0.463, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7573579549789429, + "rewards/margins": 1.0562195777893066, + "rewards/rejected": -2.81357741355896, + "step": 10600 + }, + { + "epoch": 0.69, + "eval_logits/chosen": -1.006567358970642, + "eval_logits/rejected": -0.8792454600334167, + "eval_logps/chosen": -427.3688659667969, + "eval_logps/rejected": -517.7098999023438, + "eval_loss": 0.49114155769348145, + "eval_rewards/accuracies": 0.6654999852180481, + "eval_rewards/chosen": -1.9536391496658325, + "eval_rewards/margins": 1.1073405742645264, + "eval_rewards/rejected": -3.0609798431396484, + "eval_runtime": 736.8596, + "eval_samples_per_second": 2.714, + "eval_steps_per_second": 1.357, + "step": 10600 + }, + { + "epoch": 0.69, + "learning_rate": 1.29426506304615e-06, + "logits/chosen": -0.9439483880996704, + "logits/rejected": -0.9378741979598999, + "logps/chosen": -457.62164306640625, + "logps/rejected": -493.03228759765625, + "loss": 0.7532, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.348982095718384, + "rewards/margins": 0.548886775970459, + "rewards/rejected": -2.8978688716888428, + "step": 10610 + }, + { + "epoch": 0.69, + "learning_rate": 1.289266276967855e-06, + "logits/chosen": -1.2154090404510498, + "logits/rejected": -0.9938980340957642, + "logps/chosen": -554.221435546875, + "logps/rejected": -565.4821166992188, + "loss": 0.4864, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.156395435333252, + "rewards/margins": 1.0141894817352295, + "rewards/rejected": -3.1705851554870605, + "step": 10620 + }, + { + "epoch": 0.7, + "learning_rate": 1.284273806669745e-06, + "logits/chosen": -0.9447933435440063, + "logits/rejected": -0.9030729532241821, + "logps/chosen": -509.0013122558594, + "logps/rejected": -633.0047607421875, + "loss": 0.4851, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.696763753890991, + "rewards/margins": 1.0646278858184814, + "rewards/rejected": -3.7613918781280518, + "step": 10630 + }, + { + "epoch": 0.7, + "learning_rate": 1.2792876781949884e-06, + "logits/chosen": -0.6062985062599182, + "logits/rejected": -0.521242618560791, + "logps/chosen": -408.0526428222656, + "logps/rejected": -471.83612060546875, + "loss": 0.542, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.0197997093200684, + "rewards/margins": 0.8310112953186035, + "rewards/rejected": -2.850811243057251, + "step": 10640 + }, + { + "epoch": 0.7, + "learning_rate": 1.274307917553676e-06, + "logits/chosen": -0.9177217483520508, + "logits/rejected": -0.7457525730133057, + "logps/chosen": -424.8282165527344, + "logps/rejected": -578.0344848632812, + "loss": 0.4418, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.291694402694702, + "rewards/margins": 1.2860668897628784, + "rewards/rejected": -3.57776141166687, + "step": 10650 + }, + { + "epoch": 0.7, + "learning_rate": 1.2693345507226767e-06, + "logits/chosen": -0.9302674531936646, + "logits/rejected": -0.7708099484443665, + "logps/chosen": -467.2933654785156, + "logps/rejected": -583.9351806640625, + "loss": 0.4791, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3730309009552, + "rewards/margins": 1.1452833414077759, + "rewards/rejected": -3.5183138847351074, + "step": 10660 + }, + { + "epoch": 0.7, + "learning_rate": 1.2643676036455099e-06, + "logits/chosen": -1.2523701190948486, + "logits/rejected": -1.0938878059387207, + "logps/chosen": -498.78857421875, + "logps/rejected": -543.588134765625, + "loss": 0.5088, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.109360933303833, + "rewards/margins": 0.9554061889648438, + "rewards/rejected": -3.064767360687256, + "step": 10670 + }, + { + "epoch": 0.7, + "learning_rate": 1.259407102232203e-06, + "logits/chosen": -1.209685206413269, + "logits/rejected": -0.7783905267715454, + "logps/chosen": -513.1278076171875, + "logps/rejected": -560.0278930664062, + "loss": 0.4174, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3549351692199707, + "rewards/margins": 1.1984398365020752, + "rewards/rejected": -3.553374767303467, + "step": 10680 + }, + { + "epoch": 0.7, + "learning_rate": 1.254453072359163e-06, + "logits/chosen": -0.7804247140884399, + "logits/rejected": -0.7799268364906311, + "logps/chosen": -481.41668701171875, + "logps/rejected": -561.271484375, + "loss": 0.4861, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.473966121673584, + "rewards/margins": 1.0172109603881836, + "rewards/rejected": -3.4911770820617676, + "step": 10690 + }, + { + "epoch": 0.7, + "learning_rate": 1.2495055398690337e-06, + "logits/chosen": -1.2695982456207275, + "logits/rejected": -1.037883996963501, + "logps/chosen": -449.74151611328125, + "logps/rejected": -520.7718505859375, + "loss": 0.5865, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.19887113571167, + "rewards/margins": 0.8031356930732727, + "rewards/rejected": -3.002007007598877, + "step": 10700 + }, + { + "epoch": 0.7, + "eval_logits/chosen": -0.882703423500061, + "eval_logits/rejected": -0.762727677822113, + "eval_logps/chosen": -458.7840576171875, + "eval_logps/rejected": -548.8290405273438, + "eval_loss": 0.4880741238594055, + "eval_rewards/accuracies": 0.6679999828338623, + "eval_rewards/chosen": -2.2677907943725586, + "eval_rewards/margins": 1.104380488395691, + "eval_rewards/rejected": -3.372171401977539, + "eval_runtime": 733.7904, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 10700 + }, + { + "epoch": 0.7, + "learning_rate": 1.2445645305705718e-06, + "logits/chosen": -1.0357564687728882, + "logits/rejected": -1.0038548707962036, + "logps/chosen": -434.9898376464844, + "logps/rejected": -511.59906005859375, + "loss": 0.5469, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.2866885662078857, + "rewards/margins": 0.9725219011306763, + "rewards/rejected": -3.2592105865478516, + "step": 10710 + }, + { + "epoch": 0.7, + "learning_rate": 1.2396300702384995e-06, + "logits/chosen": -1.0409080982208252, + "logits/rejected": -0.9370497465133667, + "logps/chosen": -509.2559509277344, + "logps/rejected": -534.8280029296875, + "loss": 0.6689, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.514479160308838, + "rewards/margins": 0.6323734521865845, + "rewards/rejected": -3.146852731704712, + "step": 10720 + }, + { + "epoch": 0.7, + "learning_rate": 1.234702184613381e-06, + "logits/chosen": -0.8630155324935913, + "logits/rejected": -0.661169171333313, + "logps/chosen": -444.38623046875, + "logps/rejected": -533.1038818359375, + "loss": 0.5089, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.28903865814209, + "rewards/margins": 0.8905321955680847, + "rewards/rejected": -3.1795706748962402, + "step": 10730 + }, + { + "epoch": 0.7, + "learning_rate": 1.2297808994014793e-06, + "logits/chosen": -1.1123378276824951, + "logits/rejected": -0.8954107165336609, + "logps/chosen": -511.7689514160156, + "logps/rejected": -571.2996826171875, + "loss": 0.5019, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.258340835571289, + "rewards/margins": 0.9442332983016968, + "rewards/rejected": -3.2025742530822754, + "step": 10740 + }, + { + "epoch": 0.7, + "learning_rate": 1.2248662402746314e-06, + "logits/chosen": -0.8245559930801392, + "logits/rejected": -0.9639986157417297, + "logps/chosen": -443.68450927734375, + "logps/rejected": -538.9236450195312, + "loss": 0.4695, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.5376784801483154, + "rewards/margins": 1.0135488510131836, + "rewards/rejected": -3.551227569580078, + "step": 10750 + }, + { + "epoch": 0.7, + "learning_rate": 1.2199582328701045e-06, + "logits/chosen": -1.0178649425506592, + "logits/rejected": -0.916810154914856, + "logps/chosen": -500.6128845214844, + "logps/rejected": -595.4609375, + "loss": 0.4223, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0997936725616455, + "rewards/margins": 1.3513892889022827, + "rewards/rejected": -3.4511828422546387, + "step": 10760 + }, + { + "epoch": 0.7, + "learning_rate": 1.2150569027904712e-06, + "logits/chosen": -0.9637805223464966, + "logits/rejected": -0.9385682940483093, + "logps/chosen": -465.7509765625, + "logps/rejected": -568.2579345703125, + "loss": 0.4797, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1976747512817383, + "rewards/margins": 1.0880988836288452, + "rewards/rejected": -3.285773754119873, + "step": 10770 + }, + { + "epoch": 0.71, + "learning_rate": 1.2101622756034688e-06, + "logits/chosen": -0.9674359560012817, + "logits/rejected": -0.918204665184021, + "logps/chosen": -409.08953857421875, + "logps/rejected": -481.28851318359375, + "loss": 0.493, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.8633549213409424, + "rewards/margins": 1.0256426334381104, + "rewards/rejected": -2.8889975547790527, + "step": 10780 + }, + { + "epoch": 0.71, + "learning_rate": 1.2052743768418715e-06, + "logits/chosen": -0.9215058088302612, + "logits/rejected": -0.7507846355438232, + "logps/chosen": -466.75250244140625, + "logps/rejected": -534.5064086914062, + "loss": 0.5046, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.189434289932251, + "rewards/margins": 1.0176080465316772, + "rewards/rejected": -3.2070422172546387, + "step": 10790 + }, + { + "epoch": 0.71, + "learning_rate": 1.2003932320033523e-06, + "logits/chosen": -0.9605894088745117, + "logits/rejected": -0.9722213745117188, + "logps/chosen": -460.3763122558594, + "logps/rejected": -618.0411987304688, + "loss": 0.3972, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.3387362957000732, + "rewards/margins": 1.570330023765564, + "rewards/rejected": -3.9090659618377686, + "step": 10800 + }, + { + "epoch": 0.71, + "eval_logits/chosen": -0.9055193662643433, + "eval_logits/rejected": -0.7830135226249695, + "eval_logps/chosen": -468.3777770996094, + "eval_logps/rejected": -560.4705810546875, + "eval_loss": 0.4904264211654663, + "eval_rewards/accuracies": 0.6690000295639038, + "eval_rewards/chosen": -2.3637280464172363, + "eval_rewards/margins": 1.1248583793640137, + "eval_rewards/rejected": -3.488586187362671, + "eval_runtime": 733.0719, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 10800 + }, + { + "epoch": 0.71, + "learning_rate": 1.1955188665503553e-06, + "logits/chosen": -0.8003166913986206, + "logits/rejected": -0.7389184236526489, + "logps/chosen": -425.4422302246094, + "logps/rejected": -509.4952697753906, + "loss": 0.4912, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.2031853199005127, + "rewards/margins": 1.048431396484375, + "rewards/rejected": -3.2516167163848877, + "step": 10810 + }, + { + "epoch": 0.71, + "learning_rate": 1.1906513059099566e-06, + "logits/chosen": -1.0434842109680176, + "logits/rejected": -0.7776933908462524, + "logps/chosen": -487.75811767578125, + "logps/rejected": -602.5443725585938, + "loss": 0.4286, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.562689781188965, + "rewards/margins": 1.2352688312530518, + "rewards/rejected": -3.7979588508605957, + "step": 10820 + }, + { + "epoch": 0.71, + "learning_rate": 1.185790575473738e-06, + "logits/chosen": -0.9918256998062134, + "logits/rejected": -0.6145702004432678, + "logps/chosen": -465.2567443847656, + "logps/rejected": -522.7682495117188, + "loss": 0.5531, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.361233711242676, + "rewards/margins": 1.0478578805923462, + "rewards/rejected": -3.4090912342071533, + "step": 10830 + }, + { + "epoch": 0.71, + "learning_rate": 1.1809367005976516e-06, + "logits/chosen": -0.9616907238960266, + "logits/rejected": -0.8075451850891113, + "logps/chosen": -500.2339782714844, + "logps/rejected": -505.9252014160156, + "loss": 0.5588, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.177391767501831, + "rewards/margins": 0.7673858404159546, + "rewards/rejected": -2.944777488708496, + "step": 10840 + }, + { + "epoch": 0.71, + "learning_rate": 1.1760897066018842e-06, + "logits/chosen": -0.9130662679672241, + "logits/rejected": -0.8023947477340698, + "logps/chosen": -424.35919189453125, + "logps/rejected": -545.6055297851562, + "loss": 0.4358, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.01318097114563, + "rewards/margins": 1.2724324464797974, + "rewards/rejected": -3.2856132984161377, + "step": 10850 + }, + { + "epoch": 0.71, + "learning_rate": 1.1712496187707327e-06, + "logits/chosen": -0.8917104005813599, + "logits/rejected": -1.089603066444397, + "logps/chosen": -464.90960693359375, + "logps/rejected": -616.8685913085938, + "loss": 0.3678, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.246053695678711, + "rewards/margins": 1.6602468490600586, + "rewards/rejected": -3.9063007831573486, + "step": 10860 + }, + { + "epoch": 0.71, + "learning_rate": 1.1664164623524646e-06, + "logits/chosen": -0.9778538942337036, + "logits/rejected": -0.7901339530944824, + "logps/chosen": -417.0609436035156, + "logps/rejected": -517.4918823242188, + "loss": 0.4386, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9868589639663696, + "rewards/margins": 1.2868831157684326, + "rewards/rejected": -3.2737419605255127, + "step": 10870 + }, + { + "epoch": 0.71, + "learning_rate": 1.1615902625591926e-06, + "logits/chosen": -1.0647779703140259, + "logits/rejected": -0.7082611322402954, + "logps/chosen": -475.00164794921875, + "logps/rejected": -568.5804443359375, + "loss": 0.4011, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.497804641723633, + "rewards/margins": 1.0299241542816162, + "rewards/rejected": -3.527729034423828, + "step": 10880 + }, + { + "epoch": 0.71, + "learning_rate": 1.156771044566738e-06, + "logits/chosen": -0.9863220453262329, + "logits/rejected": -0.8786922693252563, + "logps/chosen": -502.59735107421875, + "logps/rejected": -572.4608154296875, + "loss": 0.4652, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.3798270225524902, + "rewards/margins": 1.1308090686798096, + "rewards/rejected": -3.5106358528137207, + "step": 10890 + }, + { + "epoch": 0.71, + "learning_rate": 1.1519588335145037e-06, + "logits/chosen": -1.0075815916061401, + "logits/rejected": -1.190239667892456, + "logps/chosen": -456.916259765625, + "logps/rejected": -555.6007080078125, + "loss": 0.5572, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.4933745861053467, + "rewards/margins": 0.7998501062393188, + "rewards/rejected": -3.293224811553955, + "step": 10900 + }, + { + "epoch": 0.71, + "eval_logits/chosen": -0.8925363421440125, + "eval_logits/rejected": -0.7710062265396118, + "eval_logps/chosen": -468.0953674316406, + "eval_logps/rejected": -562.2437744140625, + "eval_loss": 0.489229679107666, + "eval_rewards/accuracies": 0.6679999828338623, + "eval_rewards/chosen": -2.360903739929199, + "eval_rewards/margins": 1.1454142332077026, + "eval_rewards/rejected": -3.5063183307647705, + "eval_runtime": 733.7383, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 10900 + }, + { + "epoch": 0.71, + "learning_rate": 1.1471536545053382e-06, + "logits/chosen": -0.9088330268859863, + "logits/rejected": -0.9032711982727051, + "logps/chosen": -407.5400390625, + "logps/rejected": -520.8419189453125, + "loss": 0.5155, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.985050916671753, + "rewards/margins": 1.057497262954712, + "rewards/rejected": -3.042548656463623, + "step": 10910 + }, + { + "epoch": 0.71, + "learning_rate": 1.1423555326054112e-06, + "logits/chosen": -0.8974703550338745, + "logits/rejected": -0.632853627204895, + "logps/chosen": -522.8529052734375, + "logps/rejected": -632.1292724609375, + "loss": 0.3658, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.400184154510498, + "rewards/margins": 1.8271236419677734, + "rewards/rejected": -4.2273077964782715, + "step": 10920 + }, + { + "epoch": 0.72, + "learning_rate": 1.1375644928440743e-06, + "logits/chosen": -1.0343987941741943, + "logits/rejected": -0.755792498588562, + "logps/chosen": -445.74774169921875, + "logps/rejected": -501.7667541503906, + "loss": 0.4438, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.0906448364257812, + "rewards/margins": 1.2352287769317627, + "rewards/rejected": -3.325873613357544, + "step": 10930 + }, + { + "epoch": 0.72, + "learning_rate": 1.1327805602137396e-06, + "logits/chosen": -1.0418864488601685, + "logits/rejected": -0.8314645886421204, + "logps/chosen": -501.97607421875, + "logps/rejected": -583.506103515625, + "loss": 0.4581, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.4323198795318604, + "rewards/margins": 1.3450803756713867, + "rewards/rejected": -3.777400255203247, + "step": 10940 + }, + { + "epoch": 0.72, + "learning_rate": 1.1280037596697426e-06, + "logits/chosen": -0.8969013094902039, + "logits/rejected": -0.643790602684021, + "logps/chosen": -492.24432373046875, + "logps/rejected": -691.2158203125, + "loss": 0.3989, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.713618516921997, + "rewards/margins": 1.6204471588134766, + "rewards/rejected": -4.3340654373168945, + "step": 10950 + }, + { + "epoch": 0.72, + "learning_rate": 1.123234116130216e-06, + "logits/chosen": -0.7682039737701416, + "logits/rejected": -0.6643818020820618, + "logps/chosen": -445.10321044921875, + "logps/rejected": -577.23291015625, + "loss": 0.5684, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.592148780822754, + "rewards/margins": 1.260753870010376, + "rewards/rejected": -3.85290265083313, + "step": 10960 + }, + { + "epoch": 0.72, + "learning_rate": 1.1184716544759553e-06, + "logits/chosen": -0.5035034418106079, + "logits/rejected": -0.47273555397987366, + "logps/chosen": -387.4788818359375, + "logps/rejected": -469.57037353515625, + "loss": 0.5691, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.221147060394287, + "rewards/margins": 0.6967689990997314, + "rewards/rejected": -2.9179160594940186, + "step": 10970 + }, + { + "epoch": 0.72, + "learning_rate": 1.1137163995502948e-06, + "logits/chosen": -1.3651089668273926, + "logits/rejected": -1.1751941442489624, + "logps/chosen": -429.91729736328125, + "logps/rejected": -511.0647888183594, + "loss": 0.482, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.0838229656219482, + "rewards/margins": 1.1384937763214111, + "rewards/rejected": -3.2223167419433594, + "step": 10980 + }, + { + "epoch": 0.72, + "learning_rate": 1.1089683761589717e-06, + "logits/chosen": -0.779492974281311, + "logits/rejected": -0.7341394424438477, + "logps/chosen": -440.05682373046875, + "logps/rejected": -590.7908935546875, + "loss": 0.4098, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.998504877090454, + "rewards/margins": 1.6964826583862305, + "rewards/rejected": -3.6949875354766846, + "step": 10990 + }, + { + "epoch": 0.72, + "learning_rate": 1.1042276090700044e-06, + "logits/chosen": -0.8709524869918823, + "logits/rejected": -0.927435576915741, + "logps/chosen": -479.05706787109375, + "logps/rejected": -579.4146728515625, + "loss": 0.6689, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.687549352645874, + "rewards/margins": 0.7018827199935913, + "rewards/rejected": -3.3894317150115967, + "step": 11000 + }, + { + "epoch": 0.72, + "eval_logits/chosen": -0.957138180732727, + "eval_logits/rejected": -0.8341361880302429, + "eval_logps/chosen": -453.06585693359375, + "eval_logps/rejected": -539.7461547851562, + "eval_loss": 0.488401859998703, + "eval_rewards/accuracies": 0.6685000061988831, + "eval_rewards/chosen": -2.210608720779419, + "eval_rewards/margins": 1.0707330703735352, + "eval_rewards/rejected": -3.281341552734375, + "eval_runtime": 733.4399, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.363, + "step": 11000 + }, + { + "epoch": 0.72, + "learning_rate": 1.0994941230135536e-06, + "logits/chosen": -1.0178345441818237, + "logits/rejected": -0.920695424079895, + "logps/chosen": -431.5406188964844, + "logps/rejected": -554.9949951171875, + "loss": 0.3399, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.9869062900543213, + "rewards/margins": 1.5951406955718994, + "rewards/rejected": -3.5820469856262207, + "step": 11010 + }, + { + "epoch": 0.72, + "learning_rate": 1.094767942681804e-06, + "logits/chosen": -1.4254045486450195, + "logits/rejected": -1.0217857360839844, + "logps/chosen": -526.99169921875, + "logps/rejected": -603.30224609375, + "loss": 0.4858, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.791226863861084, + "rewards/margins": 1.1551108360290527, + "rewards/rejected": -3.9463374614715576, + "step": 11020 + }, + { + "epoch": 0.72, + "learning_rate": 1.0900490927288248e-06, + "logits/chosen": -0.7323638200759888, + "logits/rejected": -0.7848427295684814, + "logps/chosen": -496.42279052734375, + "logps/rejected": -534.5650024414062, + "loss": 0.5168, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.3595452308654785, + "rewards/margins": 0.8761852383613586, + "rewards/rejected": -3.2357306480407715, + "step": 11030 + }, + { + "epoch": 0.72, + "learning_rate": 1.0853375977704511e-06, + "logits/chosen": -1.0628750324249268, + "logits/rejected": -0.9022421836853027, + "logps/chosen": -463.55535888671875, + "logps/rejected": -495.8919372558594, + "loss": 0.4713, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.2313079833984375, + "rewards/margins": 0.9634731411933899, + "rewards/rejected": -3.1947810649871826, + "step": 11040 + }, + { + "epoch": 0.72, + "learning_rate": 1.0806334823841466e-06, + "logits/chosen": -1.0171680450439453, + "logits/rejected": -1.157289981842041, + "logps/chosen": -465.65380859375, + "logps/rejected": -539.416748046875, + "loss": 0.7313, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.26037859916687, + "rewards/margins": 0.4435979723930359, + "rewards/rejected": -2.70397686958313, + "step": 11050 + }, + { + "epoch": 0.72, + "learning_rate": 1.0759367711088825e-06, + "logits/chosen": -0.8065532445907593, + "logits/rejected": -1.0416743755340576, + "logps/chosen": -377.97216796875, + "logps/rejected": -480.1463317871094, + "loss": 0.5569, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.941396713256836, + "rewards/margins": 0.6869693398475647, + "rewards/rejected": -2.628365993499756, + "step": 11060 + }, + { + "epoch": 0.72, + "learning_rate": 1.0712474884450056e-06, + "logits/chosen": -1.0178492069244385, + "logits/rejected": -0.8606100082397461, + "logps/chosen": -385.5984191894531, + "logps/rejected": -468.76947021484375, + "loss": 0.4454, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8276513814926147, + "rewards/margins": 1.1195788383483887, + "rewards/rejected": -2.9472298622131348, + "step": 11070 + }, + { + "epoch": 0.72, + "learning_rate": 1.066565658854112e-06, + "logits/chosen": -0.8222743272781372, + "logits/rejected": -0.9117767214775085, + "logps/chosen": -333.2332458496094, + "logps/rejected": -429.37579345703125, + "loss": 0.5184, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.054013252258301, + "rewards/margins": 0.9937500953674316, + "rewards/rejected": -3.0477633476257324, + "step": 11080 + }, + { + "epoch": 0.73, + "learning_rate": 1.0618913067589165e-06, + "logits/chosen": -1.0981779098510742, + "logits/rejected": -0.7434073686599731, + "logps/chosen": -420.64434814453125, + "logps/rejected": -498.775146484375, + "loss": 0.4303, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0482959747314453, + "rewards/margins": 1.1559075117111206, + "rewards/rejected": -3.2042033672332764, + "step": 11090 + }, + { + "epoch": 0.73, + "learning_rate": 1.0572244565431313e-06, + "logits/chosen": -0.938554584980011, + "logits/rejected": -0.9342789649963379, + "logps/chosen": -354.7508544921875, + "logps/rejected": -450.92523193359375, + "loss": 0.4435, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0933403968811035, + "rewards/margins": 0.9101314544677734, + "rewards/rejected": -3.003472089767456, + "step": 11100 + }, + { + "epoch": 0.73, + "eval_logits/chosen": -1.0134085416793823, + "eval_logits/rejected": -0.8863934874534607, + "eval_logps/chosen": -443.8869323730469, + "eval_logps/rejected": -533.0964965820312, + "eval_loss": 0.4877125322818756, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -2.11881947517395, + "eval_rewards/margins": 1.0960263013839722, + "eval_rewards/rejected": -3.214846134185791, + "eval_runtime": 733.4843, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.363, + "step": 11100 + }, + { + "epoch": 0.73, + "learning_rate": 1.0525651325513317e-06, + "logits/chosen": -1.0130351781845093, + "logits/rejected": -1.0235871076583862, + "logps/chosen": -542.2921142578125, + "logps/rejected": -603.1007080078125, + "loss": 0.6379, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.1102242469787598, + "rewards/margins": 0.8196399807929993, + "rewards/rejected": -2.9298644065856934, + "step": 11110 + }, + { + "epoch": 0.73, + "learning_rate": 1.0479133590888351e-06, + "logits/chosen": -1.0382142066955566, + "logits/rejected": -0.9209200739860535, + "logps/chosen": -472.59283447265625, + "logps/rejected": -546.9326171875, + "loss": 0.4894, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.1713783740997314, + "rewards/margins": 1.0131199359893799, + "rewards/rejected": -3.1844985485076904, + "step": 11120 + }, + { + "epoch": 0.73, + "learning_rate": 1.0432691604215695e-06, + "logits/chosen": -1.076169729232788, + "logits/rejected": -0.8656023144721985, + "logps/chosen": -429.9717712402344, + "logps/rejected": -481.2691345214844, + "loss": 0.5108, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9492483139038086, + "rewards/margins": 0.8301043510437012, + "rewards/rejected": -2.7793526649475098, + "step": 11130 + }, + { + "epoch": 0.73, + "learning_rate": 1.0386325607759515e-06, + "logits/chosen": -1.0083991289138794, + "logits/rejected": -0.793779730796814, + "logps/chosen": -356.7492370605469, + "logps/rejected": -465.8247985839844, + "loss": 0.4141, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6535911560058594, + "rewards/margins": 1.291459560394287, + "rewards/rejected": -2.9450504779815674, + "step": 11140 + }, + { + "epoch": 0.73, + "learning_rate": 1.0340035843387544e-06, + "logits/chosen": -0.9116052389144897, + "logits/rejected": -0.7750101089477539, + "logps/chosen": -363.89166259765625, + "logps/rejected": -446.844482421875, + "loss": 0.524, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8415607213974, + "rewards/margins": 1.1053569316864014, + "rewards/rejected": -2.94691801071167, + "step": 11150 + }, + { + "epoch": 0.73, + "learning_rate": 1.0293822552569887e-06, + "logits/chosen": -1.178986668586731, + "logits/rejected": -1.0037364959716797, + "logps/chosen": -461.3528747558594, + "logps/rejected": -518.0894775390625, + "loss": 0.573, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.0946905612945557, + "rewards/margins": 1.1256706714630127, + "rewards/rejected": -3.220360517501831, + "step": 11160 + }, + { + "epoch": 0.73, + "learning_rate": 1.0247685976377688e-06, + "logits/chosen": -1.084922432899475, + "logits/rejected": -0.8008192181587219, + "logps/chosen": -388.384765625, + "logps/rejected": -457.7376403808594, + "loss": 0.5008, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.066997528076172, + "rewards/margins": 1.0818424224853516, + "rewards/rejected": -3.1488397121429443, + "step": 11170 + }, + { + "epoch": 0.73, + "learning_rate": 1.0201626355481939e-06, + "logits/chosen": -1.206965684890747, + "logits/rejected": -1.0226151943206787, + "logps/chosen": -425.4285583496094, + "logps/rejected": -488.95074462890625, + "loss": 0.3881, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0930800437927246, + "rewards/margins": 1.1708210706710815, + "rewards/rejected": -3.2639012336730957, + "step": 11180 + }, + { + "epoch": 0.73, + "learning_rate": 1.0155643930152192e-06, + "logits/chosen": -1.2319376468658447, + "logits/rejected": -1.0966084003448486, + "logps/chosen": -492.85614013671875, + "logps/rejected": -529.0089111328125, + "loss": 0.5333, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.2024879455566406, + "rewards/margins": 0.9079867601394653, + "rewards/rejected": -3.1104745864868164, + "step": 11190 + }, + { + "epoch": 0.73, + "learning_rate": 1.0109738940255286e-06, + "logits/chosen": -0.9945646524429321, + "logits/rejected": -0.8503836393356323, + "logps/chosen": -413.6936950683594, + "logps/rejected": -578.9022216796875, + "loss": 0.5282, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9671112298965454, + "rewards/margins": 2.0248970985412598, + "rewards/rejected": -3.9920082092285156, + "step": 11200 + }, + { + "epoch": 0.73, + "eval_logits/chosen": -1.0309162139892578, + "eval_logits/rejected": -0.902747392654419, + "eval_logps/chosen": -437.6730651855469, + "eval_logps/rejected": -526.853515625, + "eval_loss": 0.487095445394516, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -2.0566811561584473, + "eval_rewards/margins": 1.0957354307174683, + "eval_rewards/rejected": -3.152416229248047, + "eval_runtime": 736.8631, + "eval_samples_per_second": 2.714, + "eval_steps_per_second": 1.357, + "step": 11200 + }, + { + "epoch": 0.73, + "learning_rate": 1.0063911625254155e-06, + "logits/chosen": -1.1311531066894531, + "logits/rejected": -1.1501240730285645, + "logps/chosen": -399.01885986328125, + "logps/rejected": -517.2718505859375, + "loss": 0.4506, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.6309621334075928, + "rewards/margins": 1.1762351989746094, + "rewards/rejected": -2.8071975708007812, + "step": 11210 + }, + { + "epoch": 0.73, + "learning_rate": 1.0018162224206502e-06, + "logits/chosen": -0.9788483381271362, + "logits/rejected": -0.9975950121879578, + "logps/chosen": -379.59552001953125, + "logps/rejected": -536.1707763671875, + "loss": 0.4089, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0512068271636963, + "rewards/margins": 1.5286515951156616, + "rewards/rejected": -3.5798587799072266, + "step": 11220 + }, + { + "epoch": 0.73, + "learning_rate": 9.97249097576363e-07, + "logits/chosen": -1.4967749118804932, + "logits/rejected": -1.068502426147461, + "logps/chosen": -436.02069091796875, + "logps/rejected": -541.9471435546875, + "loss": 0.4054, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.0296413898468018, + "rewards/margins": 1.494182825088501, + "rewards/rejected": -3.5238239765167236, + "step": 11230 + }, + { + "epoch": 0.74, + "learning_rate": 9.92689811816913e-07, + "logits/chosen": -1.0827951431274414, + "logits/rejected": -0.7837275862693787, + "logps/chosen": -419.0982360839844, + "logps/rejected": -461.9170837402344, + "loss": 0.5667, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.0657706260681152, + "rewards/margins": 0.9022183418273926, + "rewards/rejected": -2.967989206314087, + "step": 11240 + }, + { + "epoch": 0.74, + "learning_rate": 9.881383889257691e-07, + "logits/chosen": -1.017508864402771, + "logits/rejected": -1.1153932809829712, + "logps/chosen": -382.99005126953125, + "logps/rejected": -534.8026123046875, + "loss": 0.5895, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1369316577911377, + "rewards/margins": 0.8773406744003296, + "rewards/rejected": -3.014272451400757, + "step": 11250 + }, + { + "epoch": 0.74, + "learning_rate": 9.835948526453817e-07, + "logits/chosen": -0.7544322609901428, + "logits/rejected": -1.1118907928466797, + "logps/chosen": -390.35015869140625, + "logps/rejected": -519.1009521484375, + "loss": 0.5099, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.099207878112793, + "rewards/margins": 0.8507896661758423, + "rewards/rejected": -2.9499974250793457, + "step": 11260 + }, + { + "epoch": 0.74, + "learning_rate": 9.790592266770633e-07, + "logits/chosen": -1.2587848901748657, + "logits/rejected": -1.1021195650100708, + "logps/chosen": -467.99273681640625, + "logps/rejected": -533.5831298828125, + "loss": 0.4609, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.9864799976348877, + "rewards/margins": 0.938291072845459, + "rewards/rejected": -2.9247708320617676, + "step": 11270 + }, + { + "epoch": 0.74, + "learning_rate": 9.745315346808584e-07, + "logits/chosen": -0.8920543789863586, + "logits/rejected": -0.9167378544807434, + "logps/chosen": -388.6293640136719, + "logps/rejected": -438.75848388671875, + "loss": 0.6022, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.695539116859436, + "rewards/margins": 0.7251222729682922, + "rewards/rejected": -2.420661449432373, + "step": 11280 + }, + { + "epoch": 0.74, + "learning_rate": 9.70011800275428e-07, + "logits/chosen": -0.9127265214920044, + "logits/rejected": -0.8961038589477539, + "logps/chosen": -454.28143310546875, + "logps/rejected": -575.9407348632812, + "loss": 0.4861, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.16756534576416, + "rewards/margins": 1.1021320819854736, + "rewards/rejected": -3.269697666168213, + "step": 11290 + }, + { + "epoch": 0.74, + "learning_rate": 9.655000470379206e-07, + "logits/chosen": -1.0204126834869385, + "logits/rejected": -0.8335134387016296, + "logps/chosen": -408.29669189453125, + "logps/rejected": -516.9795532226562, + "loss": 0.4652, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.979491949081421, + "rewards/margins": 1.193654179573059, + "rewards/rejected": -3.1731460094451904, + "step": 11300 + }, + { + "epoch": 0.74, + "eval_logits/chosen": -1.0541530847549438, + "eval_logits/rejected": -0.925934374332428, + "eval_logps/chosen": -418.2159118652344, + "eval_logps/rejected": -505.072998046875, + "eval_loss": 0.487036794424057, + "eval_rewards/accuracies": 0.6690000295639038, + "eval_rewards/chosen": -1.8621094226837158, + "eval_rewards/margins": 1.0725016593933105, + "eval_rewards/rejected": -2.9346108436584473, + "eval_runtime": 735.7356, + "eval_samples_per_second": 2.718, + "eval_steps_per_second": 1.359, + "step": 11300 + }, + { + "epoch": 0.74, + "learning_rate": 9.609962985038517e-07, + "logits/chosen": -1.0937474966049194, + "logits/rejected": -0.9918316006660461, + "logps/chosen": -385.2691650390625, + "logps/rejected": -524.9221801757812, + "loss": 0.2925, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.6990811824798584, + "rewards/margins": 1.6477792263031006, + "rewards/rejected": -3.346860408782959, + "step": 11310 + }, + { + "epoch": 0.74, + "learning_rate": 9.565005781669786e-07, + "logits/chosen": -1.3409435749053955, + "logits/rejected": -0.9802799224853516, + "logps/chosen": -449.4964294433594, + "logps/rejected": -519.0867919921875, + "loss": 0.4207, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.8890583515167236, + "rewards/margins": 1.2290316820144653, + "rewards/rejected": -3.1180901527404785, + "step": 11320 + }, + { + "epoch": 0.74, + "learning_rate": 9.520129094791822e-07, + "logits/chosen": -0.9601613283157349, + "logits/rejected": -0.7478787302970886, + "logps/chosen": -385.11859130859375, + "logps/rejected": -475.3666076660156, + "loss": 0.5723, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.0562288761138916, + "rewards/margins": 1.0045124292373657, + "rewards/rejected": -3.0607409477233887, + "step": 11330 + }, + { + "epoch": 0.74, + "learning_rate": 9.475333158503389e-07, + "logits/chosen": -0.997717559337616, + "logits/rejected": -0.8612111806869507, + "logps/chosen": -375.82659912109375, + "logps/rejected": -420.026611328125, + "loss": 0.4935, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.5800435543060303, + "rewards/margins": 0.8935853838920593, + "rewards/rejected": -2.4736289978027344, + "step": 11340 + }, + { + "epoch": 0.74, + "learning_rate": 9.430618206482053e-07, + "logits/chosen": -0.9635284543037415, + "logits/rejected": -0.8489049077033997, + "logps/chosen": -289.9509582519531, + "logps/rejected": -372.4026794433594, + "loss": 0.5246, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.4952805042266846, + "rewards/margins": 0.8185623288154602, + "rewards/rejected": -2.3138427734375, + "step": 11350 + }, + { + "epoch": 0.74, + "learning_rate": 9.385984471982892e-07, + "logits/chosen": -0.907095730304718, + "logits/rejected": -0.7227785587310791, + "logps/chosen": -398.25634765625, + "logps/rejected": -517.0142822265625, + "loss": 0.4243, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8343048095703125, + "rewards/margins": 1.6949255466461182, + "rewards/rejected": -3.5292305946350098, + "step": 11360 + }, + { + "epoch": 0.74, + "learning_rate": 9.341432187837343e-07, + "logits/chosen": -1.2137072086334229, + "logits/rejected": -1.0345278978347778, + "logps/chosen": -358.2519836425781, + "logps/rejected": -509.11590576171875, + "loss": 0.4798, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.679481863975525, + "rewards/margins": 1.35995614528656, + "rewards/rejected": -3.039438247680664, + "step": 11370 + }, + { + "epoch": 0.74, + "learning_rate": 9.29696158645193e-07, + "logits/chosen": -1.015655755996704, + "logits/rejected": -1.135019063949585, + "logps/chosen": -419.72821044921875, + "logps/rejected": -604.85595703125, + "loss": 0.3531, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.98625910282135, + "rewards/margins": 1.5613784790039062, + "rewards/rejected": -3.547637462615967, + "step": 11380 + }, + { + "epoch": 0.75, + "learning_rate": 9.252572899807111e-07, + "logits/chosen": -1.0255063772201538, + "logits/rejected": -0.8526045083999634, + "logps/chosen": -461.0198669433594, + "logps/rejected": -576.572998046875, + "loss": 0.38, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.9042030572891235, + "rewards/margins": 1.4174002408981323, + "rewards/rejected": -3.3216030597686768, + "step": 11390 + }, + { + "epoch": 0.75, + "learning_rate": 9.208266359456003e-07, + "logits/chosen": -1.1188398599624634, + "logits/rejected": -1.008388876914978, + "logps/chosen": -364.1383056640625, + "logps/rejected": -474.17626953125, + "loss": 0.4956, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.5833252668380737, + "rewards/margins": 1.0046722888946533, + "rewards/rejected": -2.5879976749420166, + "step": 11400 + }, + { + "epoch": 0.75, + "eval_logits/chosen": -0.9939939379692078, + "eval_logits/rejected": -0.8659761548042297, + "eval_logps/chosen": -433.4949951171875, + "eval_logps/rejected": -530.9140014648438, + "eval_loss": 0.48666563630104065, + "eval_rewards/accuracies": 0.6725000143051147, + "eval_rewards/chosen": -2.0149004459381104, + "eval_rewards/margins": 1.178120732307434, + "eval_rewards/rejected": -3.193021774291992, + "eval_runtime": 732.7351, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.365, + "step": 11400 + }, + { + "epoch": 0.75, + "learning_rate": 9.164042196523229e-07, + "logits/chosen": -1.3033071756362915, + "logits/rejected": -0.9964720010757446, + "logps/chosen": -393.0906982421875, + "logps/rejected": -522.20703125, + "loss": 0.3878, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.0368967056274414, + "rewards/margins": 1.4263392686843872, + "rewards/rejected": -3.4632363319396973, + "step": 11410 + }, + { + "epoch": 0.75, + "learning_rate": 9.119900641703696e-07, + "logits/chosen": -1.2340941429138184, + "logits/rejected": -0.9759272336959839, + "logps/chosen": -451.19805908203125, + "logps/rejected": -494.56048583984375, + "loss": 0.5809, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.377516984939575, + "rewards/margins": 0.896973729133606, + "rewards/rejected": -3.2744908332824707, + "step": 11420 + }, + { + "epoch": 0.75, + "learning_rate": 9.075841925261364e-07, + "logits/chosen": -1.3645763397216797, + "logits/rejected": -1.2368052005767822, + "logps/chosen": -419.3846740722656, + "logps/rejected": -540.921142578125, + "loss": 0.4186, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.814655065536499, + "rewards/margins": 1.3596525192260742, + "rewards/rejected": -3.1743075847625732, + "step": 11430 + }, + { + "epoch": 0.75, + "learning_rate": 9.031866277028093e-07, + "logits/chosen": -0.9969218969345093, + "logits/rejected": -0.8433194160461426, + "logps/chosen": -391.59197998046875, + "logps/rejected": -549.1114501953125, + "loss": 0.4337, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.005171775817871, + "rewards/margins": 1.3070980310440063, + "rewards/rejected": -3.312269687652588, + "step": 11440 + }, + { + "epoch": 0.75, + "learning_rate": 8.987973926402391e-07, + "logits/chosen": -0.8501356244087219, + "logits/rejected": -0.9896366000175476, + "logps/chosen": -406.74560546875, + "logps/rejected": -493.015625, + "loss": 0.6122, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9440596103668213, + "rewards/margins": 0.8685744404792786, + "rewards/rejected": -2.812633991241455, + "step": 11450 + }, + { + "epoch": 0.75, + "learning_rate": 8.944165102348273e-07, + "logits/chosen": -1.1681933403015137, + "logits/rejected": -1.0174529552459717, + "logps/chosen": -300.1144104003906, + "logps/rejected": -429.978271484375, + "loss": 0.4504, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.431673288345337, + "rewards/margins": 1.2260645627975464, + "rewards/rejected": -2.657738208770752, + "step": 11460 + }, + { + "epoch": 0.75, + "learning_rate": 8.900440033394018e-07, + "logits/chosen": -0.9283140301704407, + "logits/rejected": -0.9719370603561401, + "logps/chosen": -374.1133117675781, + "logps/rejected": -455.5450134277344, + "loss": 0.5721, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9159705638885498, + "rewards/margins": 0.9071747064590454, + "rewards/rejected": -2.8231453895568848, + "step": 11470 + }, + { + "epoch": 0.75, + "learning_rate": 8.856798947631009e-07, + "logits/chosen": -1.1020092964172363, + "logits/rejected": -1.047979712486267, + "logps/chosen": -384.72314453125, + "logps/rejected": -563.4594116210938, + "loss": 0.422, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9384742975234985, + "rewards/margins": 1.6178430318832397, + "rewards/rejected": -3.5563175678253174, + "step": 11480 + }, + { + "epoch": 0.75, + "learning_rate": 8.813242072712519e-07, + "logits/chosen": -0.578697681427002, + "logits/rejected": -0.46960777044296265, + "logps/chosen": -389.91998291015625, + "logps/rejected": -459.9017639160156, + "loss": 0.642, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.2805769443511963, + "rewards/margins": 0.6884570121765137, + "rewards/rejected": -2.969034194946289, + "step": 11490 + }, + { + "epoch": 0.75, + "learning_rate": 8.769769635852557e-07, + "logits/chosen": -0.9524284601211548, + "logits/rejected": -0.9921194911003113, + "logps/chosen": -412.4534606933594, + "logps/rejected": -475.46697998046875, + "loss": 0.5636, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.9714571237564087, + "rewards/margins": 0.9977054595947266, + "rewards/rejected": -2.969162702560425, + "step": 11500 + }, + { + "epoch": 0.75, + "eval_logits/chosen": -0.988334596157074, + "eval_logits/rejected": -0.8627732396125793, + "eval_logps/chosen": -444.1772766113281, + "eval_logps/rejected": -533.0626220703125, + "eval_loss": 0.4872725009918213, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -2.1217236518859863, + "eval_rewards/margins": 1.0927841663360596, + "eval_rewards/rejected": -3.214507818222046, + "eval_runtime": 732.6713, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 11500 + }, + { + "epoch": 0.75, + "learning_rate": 8.726381863824635e-07, + "logits/chosen": -1.3175032138824463, + "logits/rejected": -1.1112231016159058, + "logps/chosen": -530.478759765625, + "logps/rejected": -558.8153076171875, + "loss": 0.5539, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.4540176391601562, + "rewards/margins": 0.957426905632019, + "rewards/rejected": -3.4114444255828857, + "step": 11510 + }, + { + "epoch": 0.75, + "learning_rate": 8.683078982960638e-07, + "logits/chosen": -0.8189307451248169, + "logits/rejected": -0.6247791647911072, + "logps/chosen": -447.772705078125, + "logps/rejected": -547.8245849609375, + "loss": 0.3892, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.23087739944458, + "rewards/margins": 1.5014287233352661, + "rewards/rejected": -3.7323060035705566, + "step": 11520 + }, + { + "epoch": 0.75, + "learning_rate": 8.639861219149584e-07, + "logits/chosen": -1.0422413349151611, + "logits/rejected": -0.782747209072113, + "logps/chosen": -491.7372131347656, + "logps/rejected": -552.7254638671875, + "loss": 0.5402, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.2775521278381348, + "rewards/margins": 1.0391275882720947, + "rewards/rejected": -3.3166794776916504, + "step": 11530 + }, + { + "epoch": 0.76, + "learning_rate": 8.596728797836532e-07, + "logits/chosen": -0.9940250515937805, + "logits/rejected": -0.9373093843460083, + "logps/chosen": -416.41790771484375, + "logps/rejected": -564.4520263671875, + "loss": 0.4311, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.011092185974121, + "rewards/margins": 1.184410810470581, + "rewards/rejected": -3.1955032348632812, + "step": 11540 + }, + { + "epoch": 0.76, + "learning_rate": 8.553681944021294e-07, + "logits/chosen": -1.1565699577331543, + "logits/rejected": -1.3328361511230469, + "logps/chosen": -427.739990234375, + "logps/rejected": -529.1275634765625, + "loss": 0.5004, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8865597248077393, + "rewards/margins": 1.2073477506637573, + "rewards/rejected": -3.0939078330993652, + "step": 11550 + }, + { + "epoch": 0.76, + "learning_rate": 8.510720882257365e-07, + "logits/chosen": -0.7032291293144226, + "logits/rejected": -0.7788906097412109, + "logps/chosen": -366.40814208984375, + "logps/rejected": -514.55224609375, + "loss": 0.437, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0510754585266113, + "rewards/margins": 1.0691219568252563, + "rewards/rejected": -3.120197057723999, + "step": 11560 + }, + { + "epoch": 0.76, + "learning_rate": 8.467845836650667e-07, + "logits/chosen": -0.613045334815979, + "logits/rejected": -0.6705749034881592, + "logps/chosen": -421.2106018066406, + "logps/rejected": -554.2785034179688, + "loss": 0.4347, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -2.1294405460357666, + "rewards/margins": 1.2651278972625732, + "rewards/rejected": -3.3945682048797607, + "step": 11570 + }, + { + "epoch": 0.76, + "learning_rate": 8.425057030858461e-07, + "logits/chosen": -0.7619770765304565, + "logits/rejected": -0.7964997291564941, + "logps/chosen": -354.86334228515625, + "logps/rejected": -490.247802734375, + "loss": 0.4928, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9880750179290771, + "rewards/margins": 1.0636155605316162, + "rewards/rejected": -3.051690101623535, + "step": 11580 + }, + { + "epoch": 0.76, + "learning_rate": 8.382354688088098e-07, + "logits/chosen": -0.8965905904769897, + "logits/rejected": -0.9813228845596313, + "logps/chosen": -345.85467529296875, + "logps/rejected": -456.6095275878906, + "loss": 0.4907, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8769928216934204, + "rewards/margins": 0.9533191919326782, + "rewards/rejected": -2.8303122520446777, + "step": 11590 + }, + { + "epoch": 0.76, + "learning_rate": 8.33973903109594e-07, + "logits/chosen": -0.9670153856277466, + "logits/rejected": -1.0012229681015015, + "logps/chosen": -440.3134765625, + "logps/rejected": -537.9239501953125, + "loss": 0.4554, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.23716402053833, + "rewards/margins": 1.291721224784851, + "rewards/rejected": -3.5288853645324707, + "step": 11600 + }, + { + "epoch": 0.76, + "eval_logits/chosen": -0.965965211391449, + "eval_logits/rejected": -0.8415982723236084, + "eval_logps/chosen": -461.8896179199219, + "eval_logps/rejected": -550.7822265625, + "eval_loss": 0.4887566864490509, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -2.298846483230591, + "eval_rewards/margins": 1.0928568840026855, + "eval_rewards/rejected": -3.3917033672332764, + "eval_runtime": 733.1262, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 11600 + }, + { + "epoch": 0.76, + "learning_rate": 8.297210282186102e-07, + "logits/chosen": -1.029415249824524, + "logits/rejected": -0.9706588983535767, + "logps/chosen": -532.1493530273438, + "logps/rejected": -625.8573608398438, + "loss": 0.6493, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.021317958831787, + "rewards/margins": 0.723414421081543, + "rewards/rejected": -3.744732618331909, + "step": 11610 + }, + { + "epoch": 0.76, + "learning_rate": 8.254768663209397e-07, + "logits/chosen": -0.818048357963562, + "logits/rejected": -0.8541353940963745, + "logps/chosen": -516.0035400390625, + "logps/rejected": -537.320068359375, + "loss": 0.4895, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.374237298965454, + "rewards/margins": 0.8378809690475464, + "rewards/rejected": -3.212118148803711, + "step": 11620 + }, + { + "epoch": 0.76, + "learning_rate": 8.212414395562079e-07, + "logits/chosen": -0.9309345483779907, + "logits/rejected": -0.8357607126235962, + "logps/chosen": -479.80535888671875, + "logps/rejected": -599.5394287109375, + "loss": 0.5408, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.4959120750427246, + "rewards/margins": 0.9581314921379089, + "rewards/rejected": -3.45404314994812, + "step": 11630 + }, + { + "epoch": 0.76, + "learning_rate": 8.170147700184775e-07, + "logits/chosen": -0.9578307867050171, + "logits/rejected": -0.887701153755188, + "logps/chosen": -495.539794921875, + "logps/rejected": -609.1138916015625, + "loss": 0.4221, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.3908846378326416, + "rewards/margins": 1.225068211555481, + "rewards/rejected": -3.615952968597412, + "step": 11640 + }, + { + "epoch": 0.76, + "learning_rate": 8.127968797561242e-07, + "logits/chosen": -1.1409074068069458, + "logits/rejected": -0.9202806353569031, + "logps/chosen": -499.4234313964844, + "logps/rejected": -620.0963134765625, + "loss": 0.4226, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.7545275688171387, + "rewards/margins": 1.430957555770874, + "rewards/rejected": -4.185485363006592, + "step": 11650 + }, + { + "epoch": 0.76, + "learning_rate": 8.085877907717338e-07, + "logits/chosen": -0.9937346577644348, + "logits/rejected": -1.119011402130127, + "logps/chosen": -436.72772216796875, + "logps/rejected": -540.747314453125, + "loss": 0.5345, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1581618785858154, + "rewards/margins": 1.1569883823394775, + "rewards/rejected": -3.315150022506714, + "step": 11660 + }, + { + "epoch": 0.76, + "learning_rate": 8.043875250219732e-07, + "logits/chosen": -0.9474736452102661, + "logits/rejected": -0.6932603120803833, + "logps/chosen": -491.7503356933594, + "logps/rejected": -531.57568359375, + "loss": 0.5792, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.6234564781188965, + "rewards/margins": 0.6089419722557068, + "rewards/rejected": -3.232398271560669, + "step": 11670 + }, + { + "epoch": 0.76, + "learning_rate": 8.001961044174881e-07, + "logits/chosen": -1.0843124389648438, + "logits/rejected": -0.6968761086463928, + "logps/chosen": -484.9103088378906, + "logps/rejected": -499.39453125, + "loss": 0.5963, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.5644583702087402, + "rewards/margins": 0.7167236804962158, + "rewards/rejected": -3.281182050704956, + "step": 11680 + }, + { + "epoch": 0.76, + "learning_rate": 7.960135508227795e-07, + "logits/chosen": -1.0498230457305908, + "logits/rejected": -0.9083242416381836, + "logps/chosen": -526.6635131835938, + "logps/rejected": -561.640625, + "loss": 0.4749, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.32794451713562, + "rewards/margins": 0.9929312467575073, + "rewards/rejected": -3.320875883102417, + "step": 11690 + }, + { + "epoch": 0.77, + "learning_rate": 7.91839886056098e-07, + "logits/chosen": -1.1805912256240845, + "logits/rejected": -0.7656027674674988, + "logps/chosen": -533.066650390625, + "logps/rejected": -621.0750732421875, + "loss": 0.4871, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.4930436611175537, + "rewards/margins": 1.185792326927185, + "rewards/rejected": -3.6788361072540283, + "step": 11700 + }, + { + "epoch": 0.77, + "eval_logits/chosen": -0.9552643299102783, + "eval_logits/rejected": -0.8321646451950073, + "eval_logps/chosen": -463.67156982421875, + "eval_logps/rejected": -548.34375, + "eval_loss": 0.49002426862716675, + "eval_rewards/accuracies": 0.6654999852180481, + "eval_rewards/chosen": -2.3166658878326416, + "eval_rewards/margins": 1.050653100013733, + "eval_rewards/rejected": -3.367318868637085, + "eval_runtime": 734.3534, + "eval_samples_per_second": 2.723, + "eval_steps_per_second": 1.362, + "step": 11700 + }, + { + "epoch": 0.77, + "learning_rate": 7.876751318893217e-07, + "logits/chosen": -0.935076892375946, + "logits/rejected": -0.7096657752990723, + "logps/chosen": -459.2938537597656, + "logps/rejected": -537.5563354492188, + "loss": 0.4755, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2142813205718994, + "rewards/margins": 0.9985069036483765, + "rewards/rejected": -3.212787628173828, + "step": 11710 + }, + { + "epoch": 0.77, + "learning_rate": 7.8351931004785e-07, + "logits/chosen": -0.6283329129219055, + "logits/rejected": -0.7397192120552063, + "logps/chosen": -445.36517333984375, + "logps/rejected": -535.31591796875, + "loss": 0.469, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.3673691749572754, + "rewards/margins": 1.1483402252197266, + "rewards/rejected": -3.515709638595581, + "step": 11720 + }, + { + "epoch": 0.77, + "learning_rate": 7.793724422104834e-07, + "logits/chosen": -1.0047307014465332, + "logits/rejected": -0.9424341917037964, + "logps/chosen": -439.0755310058594, + "logps/rejected": -636.6625366210938, + "loss": 0.4635, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.3025119304656982, + "rewards/margins": 1.2810983657836914, + "rewards/rejected": -3.5836100578308105, + "step": 11730 + }, + { + "epoch": 0.77, + "learning_rate": 7.752345500093184e-07, + "logits/chosen": -1.1682775020599365, + "logits/rejected": -1.0444492101669312, + "logps/chosen": -455.862548828125, + "logps/rejected": -490.4752502441406, + "loss": 0.6232, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.4570751190185547, + "rewards/margins": 0.6437710523605347, + "rewards/rejected": -3.100846290588379, + "step": 11740 + }, + { + "epoch": 0.77, + "learning_rate": 7.711056550296253e-07, + "logits/chosen": -1.2327778339385986, + "logits/rejected": -0.9802483320236206, + "logps/chosen": -422.8338928222656, + "logps/rejected": -508.6863708496094, + "loss": 0.4144, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7604007720947266, + "rewards/margins": 1.228775143623352, + "rewards/rejected": -2.989175796508789, + "step": 11750 + }, + { + "epoch": 0.77, + "learning_rate": 7.669857788097445e-07, + "logits/chosen": -0.6518399119377136, + "logits/rejected": -0.4853084981441498, + "logps/chosen": -377.8327331542969, + "logps/rejected": -519.7390747070312, + "loss": 0.4804, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.1626675128936768, + "rewards/margins": 1.1528141498565674, + "rewards/rejected": -3.3154819011688232, + "step": 11760 + }, + { + "epoch": 0.77, + "learning_rate": 7.628749428409676e-07, + "logits/chosen": -1.0058742761611938, + "logits/rejected": -0.8142207860946655, + "logps/chosen": -459.57562255859375, + "logps/rejected": -495.9466247558594, + "loss": 0.6283, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.2268784046173096, + "rewards/margins": 0.9512258768081665, + "rewards/rejected": -3.1781039237976074, + "step": 11770 + }, + { + "epoch": 0.77, + "learning_rate": 7.587731685674288e-07, + "logits/chosen": -1.2172415256500244, + "logits/rejected": -1.179513692855835, + "logps/chosen": -500.82598876953125, + "logps/rejected": -601.9574584960938, + "loss": 0.5644, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.272376775741577, + "rewards/margins": 0.9329449534416199, + "rewards/rejected": -3.205321788787842, + "step": 11780 + }, + { + "epoch": 0.77, + "learning_rate": 7.546804773859931e-07, + "logits/chosen": -1.156795859336853, + "logits/rejected": -1.0326814651489258, + "logps/chosen": -423.5052795410156, + "logps/rejected": -535.3846435546875, + "loss": 0.4482, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9646520614624023, + "rewards/margins": 1.3181536197662354, + "rewards/rejected": -3.2828056812286377, + "step": 11790 + }, + { + "epoch": 0.77, + "learning_rate": 7.505968906461409e-07, + "logits/chosen": -0.8699377179145813, + "logits/rejected": -1.1452893018722534, + "logps/chosen": -439.53387451171875, + "logps/rejected": -521.7918701171875, + "loss": 0.527, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.9859380722045898, + "rewards/margins": 1.0640430450439453, + "rewards/rejected": -3.049980878829956, + "step": 11800 + }, + { + "epoch": 0.77, + "eval_logits/chosen": -1.0908076763153076, + "eval_logits/rejected": -0.9602593779563904, + "eval_logps/chosen": -422.1820373535156, + "eval_logps/rejected": -508.17919921875, + "eval_loss": 0.48903682827949524, + "eval_rewards/accuracies": 0.6690000295639038, + "eval_rewards/chosen": -1.901770830154419, + "eval_rewards/margins": 1.0639021396636963, + "eval_rewards/rejected": -2.9656729698181152, + "eval_runtime": 731.3982, + "eval_samples_per_second": 2.734, + "eval_steps_per_second": 1.367, + "step": 11800 + }, + { + "epoch": 0.77, + "learning_rate": 7.465224296498627e-07, + "logits/chosen": -1.1006393432617188, + "logits/rejected": -0.8764568567276001, + "logps/chosen": -412.1211853027344, + "logps/rejected": -477.8194885253906, + "loss": 0.4437, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7916208505630493, + "rewards/margins": 1.0682731866836548, + "rewards/rejected": -2.859893798828125, + "step": 11810 + }, + { + "epoch": 0.77, + "learning_rate": 7.424571156515412e-07, + "logits/chosen": -1.1791812181472778, + "logits/rejected": -1.0099537372589111, + "logps/chosen": -355.47320556640625, + "logps/rejected": -466.53497314453125, + "loss": 0.5231, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7776803970336914, + "rewards/margins": 0.8838518261909485, + "rewards/rejected": -2.661531925201416, + "step": 11820 + }, + { + "epoch": 0.77, + "learning_rate": 7.38400969857847e-07, + "logits/chosen": -0.9811850786209106, + "logits/rejected": -0.9850279092788696, + "logps/chosen": -406.4681701660156, + "logps/rejected": -532.5477905273438, + "loss": 0.3842, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.233257532119751, + "rewards/margins": 1.229274868965149, + "rewards/rejected": -3.4625325202941895, + "step": 11830 + }, + { + "epoch": 0.77, + "learning_rate": 7.343540134276225e-07, + "logits/chosen": -1.1172263622283936, + "logits/rejected": -1.1144663095474243, + "logps/chosen": -332.30999755859375, + "logps/rejected": -423.35772705078125, + "loss": 0.578, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6159842014312744, + "rewards/margins": 0.8915143013000488, + "rewards/rejected": -2.5074985027313232, + "step": 11840 + }, + { + "epoch": 0.78, + "learning_rate": 7.303162674717762e-07, + "logits/chosen": -0.7593609094619751, + "logits/rejected": -0.6226508617401123, + "logps/chosen": -429.37359619140625, + "logps/rejected": -464.8836975097656, + "loss": 0.53, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.1914584636688232, + "rewards/margins": 0.913454532623291, + "rewards/rejected": -3.1049129962921143, + "step": 11850 + }, + { + "epoch": 0.78, + "learning_rate": 7.26287753053167e-07, + "logits/chosen": -1.109512209892273, + "logits/rejected": -0.941886305809021, + "logps/chosen": -501.8273010253906, + "logps/rejected": -605.0291137695312, + "loss": 0.5333, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.368511438369751, + "rewards/margins": 1.0447001457214355, + "rewards/rejected": -3.4132113456726074, + "step": 11860 + }, + { + "epoch": 0.78, + "learning_rate": 7.222684911865013e-07, + "logits/chosen": -1.0845589637756348, + "logits/rejected": -1.0413366556167603, + "logps/chosen": -369.25897216796875, + "logps/rejected": -516.8199462890625, + "loss": 0.4041, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.6209027767181396, + "rewards/margins": 1.258754849433899, + "rewards/rejected": -2.879657745361328, + "step": 11870 + }, + { + "epoch": 0.78, + "learning_rate": 7.182585028382166e-07, + "logits/chosen": -1.2062350511550903, + "logits/rejected": -1.0037099123001099, + "logps/chosen": -459.37249755859375, + "logps/rejected": -541.9036865234375, + "loss": 0.3956, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.8322092294692993, + "rewards/margins": 1.1498112678527832, + "rewards/rejected": -2.982020616531372, + "step": 11880 + }, + { + "epoch": 0.78, + "learning_rate": 7.142578089263769e-07, + "logits/chosen": -1.2526991367340088, + "logits/rejected": -1.2502632141113281, + "logps/chosen": -507.63128662109375, + "logps/rejected": -593.0853881835938, + "loss": 0.4494, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8920265436172485, + "rewards/margins": 1.4425415992736816, + "rewards/rejected": -3.3345680236816406, + "step": 11890 + }, + { + "epoch": 0.78, + "learning_rate": 7.102664303205611e-07, + "logits/chosen": -0.9701188802719116, + "logits/rejected": -1.1144049167633057, + "logps/chosen": -441.10723876953125, + "logps/rejected": -530.4204711914062, + "loss": 0.569, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.20544695854187, + "rewards/margins": 1.0718988180160522, + "rewards/rejected": -3.277346134185791, + "step": 11900 + }, + { + "epoch": 0.78, + "eval_logits/chosen": -1.0340651273727417, + "eval_logits/rejected": -0.9051576852798462, + "eval_logps/chosen": -439.36798095703125, + "eval_logps/rejected": -531.2297973632812, + "eval_loss": 0.48876819014549255, + "eval_rewards/accuracies": 0.6669999957084656, + "eval_rewards/chosen": -2.0736303329467773, + "eval_rewards/margins": 1.1225486993789673, + "eval_rewards/rejected": -3.196179151535034, + "eval_runtime": 732.9071, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.364, + "step": 11900 + }, + { + "epoch": 0.78, + "learning_rate": 7.062843878417566e-07, + "logits/chosen": -1.3267271518707275, + "logits/rejected": -1.1309109926223755, + "logps/chosen": -415.145751953125, + "logps/rejected": -485.17266845703125, + "loss": 0.5037, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.9171311855316162, + "rewards/margins": 1.0307111740112305, + "rewards/rejected": -2.947842836380005, + "step": 11910 + }, + { + "epoch": 0.78, + "learning_rate": 7.023117022622458e-07, + "logits/chosen": -1.1064445972442627, + "logits/rejected": -0.793212890625, + "logps/chosen": -508.7882385253906, + "logps/rejected": -580.7551879882812, + "loss": 0.5068, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.7027292251586914, + "rewards/margins": 0.9523531794548035, + "rewards/rejected": -3.6550827026367188, + "step": 11920 + }, + { + "epoch": 0.78, + "learning_rate": 6.983483943055042e-07, + "logits/chosen": -1.0946762561798096, + "logits/rejected": -0.9343063235282898, + "logps/chosen": -491.595947265625, + "logps/rejected": -513.9833984375, + "loss": 0.5994, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.1112940311431885, + "rewards/margins": 0.791604220867157, + "rewards/rejected": -2.9028983116149902, + "step": 11930 + }, + { + "epoch": 0.78, + "learning_rate": 6.943944846460859e-07, + "logits/chosen": -0.8383496403694153, + "logits/rejected": -0.8814895749092102, + "logps/chosen": -407.0166931152344, + "logps/rejected": -453.33123779296875, + "loss": 0.5948, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.8826236724853516, + "rewards/margins": 0.9305664896965027, + "rewards/rejected": -2.81318998336792, + "step": 11940 + }, + { + "epoch": 0.78, + "learning_rate": 6.904499939095225e-07, + "logits/chosen": -1.0897403955459595, + "logits/rejected": -1.018691062927246, + "logps/chosen": -407.6357421875, + "logps/rejected": -522.668212890625, + "loss": 0.4682, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8528856039047241, + "rewards/margins": 1.3036080598831177, + "rewards/rejected": -3.156493663787842, + "step": 11950 + }, + { + "epoch": 0.78, + "learning_rate": 6.865149426722079e-07, + "logits/chosen": -0.8987745046615601, + "logits/rejected": -0.8762083053588867, + "logps/chosen": -502.9815368652344, + "logps/rejected": -570.9379272460938, + "loss": 0.5286, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.3179638385772705, + "rewards/margins": 0.9865046739578247, + "rewards/rejected": -3.3044686317443848, + "step": 11960 + }, + { + "epoch": 0.78, + "learning_rate": 6.825893514612985e-07, + "logits/chosen": -0.8229151964187622, + "logits/rejected": -0.7518579363822937, + "logps/chosen": -412.1634216308594, + "logps/rejected": -548.7064208984375, + "loss": 0.4418, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7559655904769897, + "rewards/margins": 1.3872650861740112, + "rewards/rejected": -3.14323091506958, + "step": 11970 + }, + { + "epoch": 0.78, + "learning_rate": 6.786732407546001e-07, + "logits/chosen": -0.8534737825393677, + "logits/rejected": -0.7331782579421997, + "logps/chosen": -396.0401611328125, + "logps/rejected": -448.16082763671875, + "loss": 0.5383, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.951589584350586, + "rewards/margins": 0.9183629751205444, + "rewards/rejected": -2.869952440261841, + "step": 11980 + }, + { + "epoch": 0.78, + "learning_rate": 6.747666309804654e-07, + "logits/chosen": -1.3171064853668213, + "logits/rejected": -1.0033562183380127, + "logps/chosen": -522.2742919921875, + "logps/rejected": -562.9515991210938, + "loss": 0.3579, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -2.345970392227173, + "rewards/margins": 1.202128291130066, + "rewards/rejected": -3.5480988025665283, + "step": 11990 + }, + { + "epoch": 0.79, + "learning_rate": 6.708695425176831e-07, + "logits/chosen": -0.9916369318962097, + "logits/rejected": -0.7585171461105347, + "logps/chosen": -394.0176696777344, + "logps/rejected": -539.1189575195312, + "loss": 0.4233, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.249788999557495, + "rewards/margins": 1.2434598207473755, + "rewards/rejected": -3.493248701095581, + "step": 12000 + }, + { + "epoch": 0.79, + "eval_logits/chosen": -1.0466409921646118, + "eval_logits/rejected": -0.9172680377960205, + "eval_logps/chosen": -441.6598815917969, + "eval_logps/rejected": -530.766357421875, + "eval_loss": 0.48879528045654297, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -2.0965492725372314, + "eval_rewards/margins": 1.0949954986572266, + "eval_rewards/rejected": -3.191545009613037, + "eval_runtime": 730.9157, + "eval_samples_per_second": 2.736, + "eval_steps_per_second": 1.368, + "step": 12000 + }, + { + "epoch": 0.79, + "learning_rate": 6.669819956953768e-07, + "logits/chosen": -0.8059449195861816, + "logits/rejected": -0.8757872581481934, + "logps/chosen": -352.72723388671875, + "logps/rejected": -473.0796813964844, + "loss": 0.4025, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.823960542678833, + "rewards/margins": 1.118124008178711, + "rewards/rejected": -2.942084550857544, + "step": 12010 + }, + { + "epoch": 0.79, + "learning_rate": 6.631040107928957e-07, + "logits/chosen": -1.3854398727416992, + "logits/rejected": -0.8418337106704712, + "logps/chosen": -502.0740661621094, + "logps/rejected": -510.20123291015625, + "loss": 0.4322, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.338897228240967, + "rewards/margins": 1.0442569255828857, + "rewards/rejected": -3.3831546306610107, + "step": 12020 + }, + { + "epoch": 0.79, + "learning_rate": 6.592356080397072e-07, + "logits/chosen": -1.0946362018585205, + "logits/rejected": -0.8409433364868164, + "logps/chosen": -424.310546875, + "logps/rejected": -497.91796875, + "loss": 0.4945, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9602136611938477, + "rewards/margins": 1.300708293914795, + "rewards/rejected": -3.2609219551086426, + "step": 12030 + }, + { + "epoch": 0.79, + "learning_rate": 6.553768076152963e-07, + "logits/chosen": -0.8674459457397461, + "logits/rejected": -1.0205134153366089, + "logps/chosen": -333.92681884765625, + "logps/rejected": -509.59979248046875, + "loss": 0.4385, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7582921981811523, + "rewards/margins": 1.4856898784637451, + "rewards/rejected": -3.2439818382263184, + "step": 12040 + }, + { + "epoch": 0.79, + "learning_rate": 6.51527629649055e-07, + "logits/chosen": -1.3031247854232788, + "logits/rejected": -1.123964786529541, + "logps/chosen": -489.2030334472656, + "logps/rejected": -507.7479553222656, + "loss": 0.6789, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.356199264526367, + "rewards/margins": 0.47692030668258667, + "rewards/rejected": -2.8331196308135986, + "step": 12050 + }, + { + "epoch": 0.79, + "learning_rate": 6.476880942201824e-07, + "logits/chosen": -1.491958498954773, + "logits/rejected": -1.0982462167739868, + "logps/chosen": -372.901611328125, + "logps/rejected": -446.5074768066406, + "loss": 0.4808, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.3156394958496094, + "rewards/margins": 1.3223001956939697, + "rewards/rejected": -2.637939929962158, + "step": 12060 + }, + { + "epoch": 0.79, + "learning_rate": 6.438582213575748e-07, + "logits/chosen": -1.1286519765853882, + "logits/rejected": -1.1970762014389038, + "logps/chosen": -405.6150207519531, + "logps/rejected": -532.4569091796875, + "loss": 0.4548, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7569844722747803, + "rewards/margins": 1.1569640636444092, + "rewards/rejected": -2.9139487743377686, + "step": 12070 + }, + { + "epoch": 0.79, + "learning_rate": 6.400380310397267e-07, + "logits/chosen": -1.1258952617645264, + "logits/rejected": -0.8831712007522583, + "logps/chosen": -435.85418701171875, + "logps/rejected": -529.7272338867188, + "loss": 0.7571, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.037438154220581, + "rewards/margins": 0.6246122121810913, + "rewards/rejected": -2.662050724029541, + "step": 12080 + }, + { + "epoch": 0.79, + "learning_rate": 6.362275431946202e-07, + "logits/chosen": -0.9884549975395203, + "logits/rejected": -1.0082800388336182, + "logps/chosen": -429.74102783203125, + "logps/rejected": -526.2401123046875, + "loss": 0.5738, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9338738918304443, + "rewards/margins": 0.904509425163269, + "rewards/rejected": -2.838383436203003, + "step": 12090 + }, + { + "epoch": 0.79, + "learning_rate": 6.324267776996285e-07, + "logits/chosen": -1.4050889015197754, + "logits/rejected": -0.9676424264907837, + "logps/chosen": -574.62158203125, + "logps/rejected": -605.3466796875, + "loss": 0.3903, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.00042462348938, + "rewards/margins": 1.5270559787750244, + "rewards/rejected": -3.527480363845825, + "step": 12100 + }, + { + "epoch": 0.79, + "eval_logits/chosen": -1.1908217668533325, + "eval_logits/rejected": -1.056341528892517, + "eval_logps/chosen": -398.17730712890625, + "eval_logps/rejected": -481.9284973144531, + "eval_loss": 0.49027806520462036, + "eval_rewards/accuracies": 0.6664999723434448, + "eval_rewards/chosen": -1.6617236137390137, + "eval_rewards/margins": 1.0414423942565918, + "eval_rewards/rejected": -2.7031657695770264, + "eval_runtime": 732.5323, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 12100 + }, + { + "epoch": 0.79, + "learning_rate": 6.286357543814045e-07, + "logits/chosen": -1.0828051567077637, + "logits/rejected": -1.1512118577957153, + "logps/chosen": -353.3865051269531, + "logps/rejected": -540.902099609375, + "loss": 0.4649, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.5985664129257202, + "rewards/margins": 1.1950440406799316, + "rewards/rejected": -2.7936103343963623, + "step": 12110 + }, + { + "epoch": 0.79, + "learning_rate": 6.248544930157838e-07, + "logits/chosen": -1.2649319171905518, + "logits/rejected": -1.0828841924667358, + "logps/chosen": -356.8739929199219, + "logps/rejected": -485.9205627441406, + "loss": 0.4563, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.7278845310211182, + "rewards/margins": 1.4682705402374268, + "rewards/rejected": -3.196155071258545, + "step": 12120 + }, + { + "epoch": 0.79, + "learning_rate": 6.21083013327678e-07, + "logits/chosen": -1.1251111030578613, + "logits/rejected": -1.065676212310791, + "logps/chosen": -455.14129638671875, + "logps/rejected": -474.03887939453125, + "loss": 0.5947, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.5090054273605347, + "rewards/margins": 0.748233437538147, + "rewards/rejected": -2.2572388648986816, + "step": 12130 + }, + { + "epoch": 0.79, + "learning_rate": 6.17321334990973e-07, + "logits/chosen": -1.2648893594741821, + "logits/rejected": -1.0302037000656128, + "logps/chosen": -353.55010986328125, + "logps/rejected": -410.7579650878906, + "loss": 0.5201, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.5274560451507568, + "rewards/margins": 0.9244512319564819, + "rewards/rejected": -2.4519073963165283, + "step": 12140 + }, + { + "epoch": 0.79, + "learning_rate": 6.135694776284243e-07, + "logits/chosen": -1.4515762329101562, + "logits/rejected": -1.0432875156402588, + "logps/chosen": -432.5904235839844, + "logps/rejected": -496.59246826171875, + "loss": 0.4491, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.617993950843811, + "rewards/margins": 1.2877423763275146, + "rewards/rejected": -2.9057364463806152, + "step": 12150 + }, + { + "epoch": 0.8, + "learning_rate": 6.098274608115595e-07, + "logits/chosen": -1.2649531364440918, + "logits/rejected": -0.9151164889335632, + "logps/chosen": -360.52301025390625, + "logps/rejected": -400.2120056152344, + "loss": 0.5682, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.5487239360809326, + "rewards/margins": 0.6832138895988464, + "rewards/rejected": -2.231937885284424, + "step": 12160 + }, + { + "epoch": 0.8, + "learning_rate": 6.060953040605697e-07, + "logits/chosen": -1.3290317058563232, + "logits/rejected": -0.9182616472244263, + "logps/chosen": -492.67193603515625, + "logps/rejected": -576.432373046875, + "loss": 0.4024, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.5354218482971191, + "rewards/margins": 1.536554217338562, + "rewards/rejected": -3.0719761848449707, + "step": 12170 + }, + { + "epoch": 0.8, + "learning_rate": 6.023730268442144e-07, + "logits/chosen": -1.117250680923462, + "logits/rejected": -0.9653674364089966, + "logps/chosen": -363.29437255859375, + "logps/rejected": -477.4454040527344, + "loss": 0.3805, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.584735631942749, + "rewards/margins": 1.4259586334228516, + "rewards/rejected": -3.0106940269470215, + "step": 12180 + }, + { + "epoch": 0.8, + "learning_rate": 5.986606485797131e-07, + "logits/chosen": -1.083444356918335, + "logits/rejected": -1.1372604370117188, + "logps/chosen": -383.82757568359375, + "logps/rejected": -481.3438415527344, + "loss": 0.5378, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.8393703699111938, + "rewards/margins": 0.9245020747184753, + "rewards/rejected": -2.7638726234436035, + "step": 12190 + }, + { + "epoch": 0.8, + "learning_rate": 5.949581886326511e-07, + "logits/chosen": -1.0700901746749878, + "logits/rejected": -1.1865692138671875, + "logps/chosen": -461.2632751464844, + "logps/rejected": -524.6785888671875, + "loss": 0.4775, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.6347122192382812, + "rewards/margins": 0.9232962727546692, + "rewards/rejected": -2.5580084323883057, + "step": 12200 + }, + { + "epoch": 0.8, + "eval_logits/chosen": -1.195405125617981, + "eval_logits/rejected": -1.060084581375122, + "eval_logps/chosen": -398.98553466796875, + "eval_logps/rejected": -484.2724609375, + "eval_loss": 0.48996442556381226, + "eval_rewards/accuracies": 0.6679999828338623, + "eval_rewards/chosen": -1.669805645942688, + "eval_rewards/margins": 1.0568000078201294, + "eval_rewards/rejected": -2.7266058921813965, + "eval_runtime": 732.7693, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.365, + "step": 12200 + }, + { + "epoch": 0.8, + "learning_rate": 5.912656663168717e-07, + "logits/chosen": -1.3327730894088745, + "logits/rejected": -1.2746068239212036, + "logps/chosen": -395.2479248046875, + "logps/rejected": -469.552734375, + "loss": 0.5479, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7123855352401733, + "rewards/margins": 0.8067728281021118, + "rewards/rejected": -2.5191586017608643, + "step": 12210 + }, + { + "epoch": 0.8, + "learning_rate": 5.875831008943817e-07, + "logits/chosen": -1.0541003942489624, + "logits/rejected": -1.028871774673462, + "logps/chosen": -334.6326904296875, + "logps/rejected": -380.36761474609375, + "loss": 0.6011, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.569886565208435, + "rewards/margins": 0.6833506226539612, + "rewards/rejected": -2.253237247467041, + "step": 12220 + }, + { + "epoch": 0.8, + "learning_rate": 5.839105115752442e-07, + "logits/chosen": -1.0335330963134766, + "logits/rejected": -0.9258529543876648, + "logps/chosen": -411.098876953125, + "logps/rejected": -468.7681579589844, + "loss": 0.5753, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8812135457992554, + "rewards/margins": 0.9767706990242004, + "rewards/rejected": -2.8579843044281006, + "step": 12230 + }, + { + "epoch": 0.8, + "learning_rate": 5.802479175174855e-07, + "logits/chosen": -1.0795843601226807, + "logits/rejected": -1.0609333515167236, + "logps/chosen": -330.22186279296875, + "logps/rejected": -442.25689697265625, + "loss": 0.5117, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6422760486602783, + "rewards/margins": 1.0610793828964233, + "rewards/rejected": -2.703355312347412, + "step": 12240 + }, + { + "epoch": 0.8, + "learning_rate": 5.765953378269901e-07, + "logits/chosen": -1.23228120803833, + "logits/rejected": -1.238350510597229, + "logps/chosen": -364.57135009765625, + "logps/rejected": -524.7144775390625, + "loss": 0.4668, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.575779914855957, + "rewards/margins": 1.2909927368164062, + "rewards/rejected": -2.8667726516723633, + "step": 12250 + }, + { + "epoch": 0.8, + "learning_rate": 5.729527915574037e-07, + "logits/chosen": -1.114385962486267, + "logits/rejected": -1.1761515140533447, + "logps/chosen": -394.2991943359375, + "logps/rejected": -498.36712646484375, + "loss": 0.428, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7388063669204712, + "rewards/margins": 1.0372883081436157, + "rewards/rejected": -2.776094436645508, + "step": 12260 + }, + { + "epoch": 0.8, + "learning_rate": 5.693202977100304e-07, + "logits/chosen": -1.0736769437789917, + "logits/rejected": -0.946702778339386, + "logps/chosen": -348.699462890625, + "logps/rejected": -429.6670837402344, + "loss": 0.5823, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.78554368019104, + "rewards/margins": 0.8430388569831848, + "rewards/rejected": -2.62858247756958, + "step": 12270 + }, + { + "epoch": 0.8, + "learning_rate": 5.656978752337389e-07, + "logits/chosen": -1.2007639408111572, + "logits/rejected": -1.1803643703460693, + "logps/chosen": -399.9928894042969, + "logps/rejected": -526.2418212890625, + "loss": 0.5012, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.0033774375915527, + "rewards/margins": 1.2484910488128662, + "rewards/rejected": -3.251868486404419, + "step": 12280 + }, + { + "epoch": 0.8, + "learning_rate": 5.620855430248581e-07, + "logits/chosen": -1.0226736068725586, + "logits/rejected": -1.0721031427383423, + "logps/chosen": -274.5044250488281, + "logps/rejected": -405.9659729003906, + "loss": 0.4094, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.125093698501587, + "rewards/margins": 1.336398959159851, + "rewards/rejected": -2.4614925384521484, + "step": 12290 + }, + { + "epoch": 0.8, + "learning_rate": 5.584833199270837e-07, + "logits/chosen": -1.3347752094268799, + "logits/rejected": -1.03730046749115, + "logps/chosen": -417.37310791015625, + "logps/rejected": -525.0247802734375, + "loss": 0.4513, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.8874075412750244, + "rewards/margins": 1.1534497737884521, + "rewards/rejected": -3.0408573150634766, + "step": 12300 + }, + { + "epoch": 0.8, + "eval_logits/chosen": -1.197284460067749, + "eval_logits/rejected": -1.0617622137069702, + "eval_logps/chosen": -395.2168273925781, + "eval_logps/rejected": -481.4833068847656, + "eval_loss": 0.4890175759792328, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -1.632118582725525, + "eval_rewards/margins": 1.0665956735610962, + "eval_rewards/rejected": -2.698714017868042, + "eval_runtime": 734.6568, + "eval_samples_per_second": 2.722, + "eval_steps_per_second": 1.361, + "step": 12300 + }, + { + "epoch": 0.81, + "learning_rate": 5.548912247313742e-07, + "logits/chosen": -1.559452772140503, + "logits/rejected": -1.2473337650299072, + "logps/chosen": -469.1334533691406, + "logps/rejected": -515.8812255859375, + "loss": 0.4888, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.8533347845077515, + "rewards/margins": 0.9432352185249329, + "rewards/rejected": -2.796569585800171, + "step": 12310 + }, + { + "epoch": 0.81, + "learning_rate": 5.513092761758596e-07, + "logits/chosen": -1.4145115613937378, + "logits/rejected": -1.1812801361083984, + "logps/chosen": -486.68231201171875, + "logps/rejected": -490.2225036621094, + "loss": 0.653, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.154723882675171, + "rewards/margins": 0.7052110433578491, + "rewards/rejected": -2.8599350452423096, + "step": 12320 + }, + { + "epoch": 0.81, + "learning_rate": 5.477374929457363e-07, + "logits/chosen": -1.4101641178131104, + "logits/rejected": -1.215849757194519, + "logps/chosen": -375.29510498046875, + "logps/rejected": -438.86187744140625, + "loss": 0.5445, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7323395013809204, + "rewards/margins": 0.8273865580558777, + "rewards/rejected": -2.5597259998321533, + "step": 12330 + }, + { + "epoch": 0.81, + "learning_rate": 5.441758936731772e-07, + "logits/chosen": -1.2345798015594482, + "logits/rejected": -0.9684113264083862, + "logps/chosen": -402.4441223144531, + "logps/rejected": -490.9378967285156, + "loss": 0.5247, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6709884405136108, + "rewards/margins": 1.062840223312378, + "rewards/rejected": -2.7338287830352783, + "step": 12340 + }, + { + "epoch": 0.81, + "learning_rate": 5.406244969372273e-07, + "logits/chosen": -1.2190395593643188, + "logits/rejected": -1.0561177730560303, + "logps/chosen": -358.269287109375, + "logps/rejected": -521.3367309570312, + "loss": 0.3672, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.5854642391204834, + "rewards/margins": 1.5656545162200928, + "rewards/rejected": -3.1511189937591553, + "step": 12350 + }, + { + "epoch": 0.81, + "learning_rate": 5.370833212637122e-07, + "logits/chosen": -1.0202383995056152, + "logits/rejected": -0.8779742121696472, + "logps/chosen": -389.3901672363281, + "logps/rejected": -490.498291015625, + "loss": 0.466, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.708791732788086, + "rewards/margins": 1.1489331722259521, + "rewards/rejected": -2.857724905014038, + "step": 12360 + }, + { + "epoch": 0.81, + "learning_rate": 5.335523851251392e-07, + "logits/chosen": -1.2496132850646973, + "logits/rejected": -1.1335455179214478, + "logps/chosen": -362.69232177734375, + "logps/rejected": -447.7772521972656, + "loss": 0.6054, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.565584421157837, + "rewards/margins": 1.0588310956954956, + "rewards/rejected": -2.624415397644043, + "step": 12370 + }, + { + "epoch": 0.81, + "learning_rate": 5.300317069406003e-07, + "logits/chosen": -1.2288280725479126, + "logits/rejected": -1.0688912868499756, + "logps/chosen": -311.70428466796875, + "logps/rejected": -439.04925537109375, + "loss": 0.466, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.5095841884613037, + "rewards/margins": 1.2357814311981201, + "rewards/rejected": -2.745365619659424, + "step": 12380 + }, + { + "epoch": 0.81, + "learning_rate": 5.265213050756782e-07, + "logits/chosen": -1.4298914670944214, + "logits/rejected": -1.1860029697418213, + "logps/chosen": -406.72088623046875, + "logps/rejected": -503.35589599609375, + "loss": 0.6494, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.7466964721679688, + "rewards/margins": 0.9876617193222046, + "rewards/rejected": -2.734358310699463, + "step": 12390 + }, + { + "epoch": 0.81, + "learning_rate": 5.230211978423477e-07, + "logits/chosen": -1.2988332509994507, + "logits/rejected": -1.1887037754058838, + "logps/chosen": -382.57122802734375, + "logps/rejected": -462.72625732421875, + "loss": 0.5514, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.6377102136611938, + "rewards/margins": 0.9532381296157837, + "rewards/rejected": -2.5909483432769775, + "step": 12400 + }, + { + "epoch": 0.81, + "eval_logits/chosen": -1.191559910774231, + "eval_logits/rejected": -1.0565053224563599, + "eval_logps/chosen": -392.5486145019531, + "eval_logps/rejected": -475.8312072753906, + "eval_loss": 0.4892647862434387, + "eval_rewards/accuracies": 0.6664999723434448, + "eval_rewards/chosen": -1.60543692111969, + "eval_rewards/margins": 1.0367563962936401, + "eval_rewards/rejected": -2.64219331741333, + "eval_runtime": 733.9406, + "eval_samples_per_second": 2.725, + "eval_steps_per_second": 1.363, + "step": 12400 + }, + { + "epoch": 0.81, + "learning_rate": 5.195314034988835e-07, + "logits/chosen": -1.4519113302230835, + "logits/rejected": -1.2425501346588135, + "logps/chosen": -361.20416259765625, + "logps/rejected": -404.88507080078125, + "loss": 0.495, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.4191709756851196, + "rewards/margins": 1.1072009801864624, + "rewards/rejected": -2.5263724327087402, + "step": 12410 + }, + { + "epoch": 0.81, + "learning_rate": 5.160519402497616e-07, + "logits/chosen": -1.230360746383667, + "logits/rejected": -1.1314585208892822, + "logps/chosen": -421.28509521484375, + "logps/rejected": -527.0120239257812, + "loss": 0.4999, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9123108386993408, + "rewards/margins": 1.0978678464889526, + "rewards/rejected": -3.010178804397583, + "step": 12420 + }, + { + "epoch": 0.81, + "learning_rate": 5.125828262455679e-07, + "logits/chosen": -1.1922781467437744, + "logits/rejected": -0.9991927146911621, + "logps/chosen": -407.961669921875, + "logps/rejected": -493.1849060058594, + "loss": 0.46, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.5224859714508057, + "rewards/margins": 1.155627727508545, + "rewards/rejected": -2.6781134605407715, + "step": 12430 + }, + { + "epoch": 0.81, + "learning_rate": 5.091240795828992e-07, + "logits/chosen": -0.9438712000846863, + "logits/rejected": -0.9053109288215637, + "logps/chosen": -362.1900939941406, + "logps/rejected": -483.2745056152344, + "loss": 0.453, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.596996545791626, + "rewards/margins": 1.0125372409820557, + "rewards/rejected": -2.6095337867736816, + "step": 12440 + }, + { + "epoch": 0.81, + "learning_rate": 5.056757183042732e-07, + "logits/chosen": -1.1937294006347656, + "logits/rejected": -1.0304909944534302, + "logps/chosen": -407.8619384765625, + "logps/rejected": -496.8045959472656, + "loss": 0.4515, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7430578470230103, + "rewards/margins": 1.0882017612457275, + "rewards/rejected": -2.8312594890594482, + "step": 12450 + }, + { + "epoch": 0.82, + "learning_rate": 5.022377603980308e-07, + "logits/chosen": -1.404693841934204, + "logits/rejected": -0.9854405522346497, + "logps/chosen": -406.7601013183594, + "logps/rejected": -469.7742614746094, + "loss": 0.407, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.574753999710083, + "rewards/margins": 1.2948553562164307, + "rewards/rejected": -2.8696091175079346, + "step": 12460 + }, + { + "epoch": 0.82, + "learning_rate": 4.988102237982454e-07, + "logits/chosen": -1.2445813417434692, + "logits/rejected": -1.039184331893921, + "logps/chosen": -412.42230224609375, + "logps/rejected": -452.43231201171875, + "loss": 0.5084, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9024187326431274, + "rewards/margins": 0.8058527112007141, + "rewards/rejected": -2.7082715034484863, + "step": 12470 + }, + { + "epoch": 0.82, + "learning_rate": 4.953931263846251e-07, + "logits/chosen": -1.2200262546539307, + "logits/rejected": -1.0197694301605225, + "logps/chosen": -442.448974609375, + "logps/rejected": -515.5526123046875, + "loss": 0.5027, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8074004650115967, + "rewards/margins": 1.1706429719924927, + "rewards/rejected": -2.978043556213379, + "step": 12480 + }, + { + "epoch": 0.82, + "learning_rate": 4.919864859824266e-07, + "logits/chosen": -1.1267067193984985, + "logits/rejected": -1.1097023487091064, + "logps/chosen": -420.2432556152344, + "logps/rejected": -481.21209716796875, + "loss": 0.4969, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.8280909061431885, + "rewards/margins": 1.0600405931472778, + "rewards/rejected": -2.888131618499756, + "step": 12490 + }, + { + "epoch": 0.82, + "learning_rate": 4.885903203623532e-07, + "logits/chosen": -1.5061956644058228, + "logits/rejected": -0.9625085592269897, + "logps/chosen": -440.2173767089844, + "logps/rejected": -514.4992065429688, + "loss": 0.4187, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.552034854888916, + "rewards/margins": 1.3805524110794067, + "rewards/rejected": -2.932587146759033, + "step": 12500 + }, + { + "epoch": 0.82, + "eval_logits/chosen": -1.143744945526123, + "eval_logits/rejected": -1.0092847347259521, + "eval_logps/chosen": -400.1340026855469, + "eval_logps/rejected": -489.6676330566406, + "eval_loss": 0.487697571516037, + "eval_rewards/accuracies": 0.6685000061988831, + "eval_rewards/chosen": -1.6812905073165894, + "eval_rewards/margins": 1.0992670059204102, + "eval_rewards/rejected": -2.78055739402771, + "eval_runtime": 731.5547, + "eval_samples_per_second": 2.734, + "eval_steps_per_second": 1.367, + "step": 12500 + }, + { + "epoch": 0.82, + "learning_rate": 4.852046472404695e-07, + "logits/chosen": -1.213766098022461, + "logits/rejected": -0.7593834400177002, + "logps/chosen": -455.54986572265625, + "logps/rejected": -451.78857421875, + "loss": 0.4655, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5766116380691528, + "rewards/margins": 1.219120740890503, + "rewards/rejected": -2.7957324981689453, + "step": 12510 + }, + { + "epoch": 0.82, + "learning_rate": 4.818294842781035e-07, + "logits/chosen": -1.3219027519226074, + "logits/rejected": -1.0103774070739746, + "logps/chosen": -366.6271057128906, + "logps/rejected": -465.59600830078125, + "loss": 0.4485, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.4041863679885864, + "rewards/margins": 1.5132675170898438, + "rewards/rejected": -2.9174537658691406, + "step": 12520 + }, + { + "epoch": 0.82, + "learning_rate": 4.784648490817601e-07, + "logits/chosen": -1.2231624126434326, + "logits/rejected": -1.0757906436920166, + "logps/chosen": -376.15869140625, + "logps/rejected": -442.31689453125, + "loss": 0.552, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.5605192184448242, + "rewards/margins": 1.0941671133041382, + "rewards/rejected": -2.654686689376831, + "step": 12530 + }, + { + "epoch": 0.82, + "learning_rate": 4.751107592030235e-07, + "logits/chosen": -1.202649474143982, + "logits/rejected": -0.9246581196784973, + "logps/chosen": -297.8594665527344, + "logps/rejected": -425.0567321777344, + "loss": 0.3797, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.2800767421722412, + "rewards/margins": 1.4429724216461182, + "rewards/rejected": -2.7230491638183594, + "step": 12540 + }, + { + "epoch": 0.82, + "learning_rate": 4.717672321384703e-07, + "logits/chosen": -1.1258949041366577, + "logits/rejected": -0.6755282282829285, + "logps/chosen": -390.7144470214844, + "logps/rejected": -458.26007080078125, + "loss": 0.4647, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.674687147140503, + "rewards/margins": 1.0974136590957642, + "rewards/rejected": -2.7721009254455566, + "step": 12550 + }, + { + "epoch": 0.82, + "learning_rate": 4.684342853295748e-07, + "logits/chosen": -0.9751138687133789, + "logits/rejected": -0.9520295262336731, + "logps/chosen": -335.04559326171875, + "logps/rejected": -444.92578125, + "loss": 0.5004, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.473488211631775, + "rewards/margins": 1.1380419731140137, + "rewards/rejected": -2.611530065536499, + "step": 12560 + }, + { + "epoch": 0.82, + "learning_rate": 4.651119361626213e-07, + "logits/chosen": -1.5337803363800049, + "logits/rejected": -1.0685784816741943, + "logps/chosen": -383.00390625, + "logps/rejected": -446.67694091796875, + "loss": 0.4472, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.4188528060913086, + "rewards/margins": 1.0990186929702759, + "rewards/rejected": -2.517871379852295, + "step": 12570 + }, + { + "epoch": 0.82, + "learning_rate": 4.618002019686091e-07, + "logits/chosen": -1.1882233619689941, + "logits/rejected": -1.1861683130264282, + "logps/chosen": -456.61456298828125, + "logps/rejected": -500.18994140625, + "loss": 0.6327, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.774022102355957, + "rewards/margins": 1.0021147727966309, + "rewards/rejected": -2.776136875152588, + "step": 12580 + }, + { + "epoch": 0.82, + "learning_rate": 4.5849910002316757e-07, + "logits/chosen": -1.2046785354614258, + "logits/rejected": -1.0692670345306396, + "logps/chosen": -360.4275817871094, + "logps/rejected": -454.0997009277344, + "loss": 0.4372, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.799154281616211, + "rewards/margins": 1.2693634033203125, + "rewards/rejected": -3.0685176849365234, + "step": 12590 + }, + { + "epoch": 0.82, + "learning_rate": 4.5520864754645984e-07, + "logits/chosen": -1.4719334840774536, + "logits/rejected": -1.2618463039398193, + "logps/chosen": -452.3905334472656, + "logps/rejected": -497.9466857910156, + "loss": 0.549, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7104072570800781, + "rewards/margins": 0.8938120007514954, + "rewards/rejected": -2.6042189598083496, + "step": 12600 + }, + { + "epoch": 0.82, + "eval_logits/chosen": -1.1529422998428345, + "eval_logits/rejected": -1.0171226263046265, + "eval_logps/chosen": -399.7242736816406, + "eval_logps/rejected": -491.4219665527344, + "eval_loss": 0.4873626232147217, + "eval_rewards/accuracies": 0.6694999933242798, + "eval_rewards/chosen": -1.677193522453308, + "eval_rewards/margins": 1.1209073066711426, + "eval_rewards/rejected": -2.7981014251708984, + "eval_runtime": 733.7259, + "eval_samples_per_second": 2.726, + "eval_steps_per_second": 1.363, + "step": 12600 + }, + { + "epoch": 0.83, + "learning_rate": 4.5192886170309896e-07, + "logits/chosen": -1.0846290588378906, + "logits/rejected": -1.0304473638534546, + "logps/chosen": -357.4346618652344, + "logps/rejected": -440.633544921875, + "loss": 0.6216, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6666618585586548, + "rewards/margins": 0.8301478624343872, + "rewards/rejected": -2.496809482574463, + "step": 12610 + }, + { + "epoch": 0.83, + "learning_rate": 4.486597596020548e-07, + "logits/chosen": -1.0042126178741455, + "logits/rejected": -1.0394890308380127, + "logps/chosen": -413.8369140625, + "logps/rejected": -502.90618896484375, + "loss": 0.4393, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9425557851791382, + "rewards/margins": 1.2739102840423584, + "rewards/rejected": -3.2164664268493652, + "step": 12620 + }, + { + "epoch": 0.83, + "learning_rate": 4.454013582965644e-07, + "logits/chosen": -0.9420401453971863, + "logits/rejected": -0.7326041460037231, + "logps/chosen": -453.96148681640625, + "logps/rejected": -489.3179626464844, + "loss": 0.5523, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.9050474166870117, + "rewards/margins": 0.8579981923103333, + "rewards/rejected": -2.7630457878112793, + "step": 12630 + }, + { + "epoch": 0.83, + "learning_rate": 4.4215367478404605e-07, + "logits/chosen": -1.0251177549362183, + "logits/rejected": -0.948135495185852, + "logps/chosen": -467.2874450683594, + "logps/rejected": -587.9501953125, + "loss": 0.5126, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9652748107910156, + "rewards/margins": 0.9629101753234863, + "rewards/rejected": -2.928185224533081, + "step": 12640 + }, + { + "epoch": 0.83, + "learning_rate": 4.389167260060068e-07, + "logits/chosen": -1.1111929416656494, + "logits/rejected": -1.0775830745697021, + "logps/chosen": -350.09063720703125, + "logps/rejected": -454.56695556640625, + "loss": 0.3838, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.4140043258666992, + "rewards/margins": 1.4265081882476807, + "rewards/rejected": -2.84051251411438, + "step": 12650 + }, + { + "epoch": 0.83, + "learning_rate": 4.356905288479579e-07, + "logits/chosen": -1.1360782384872437, + "logits/rejected": -0.882355809211731, + "logps/chosen": -378.951416015625, + "logps/rejected": -504.17218017578125, + "loss": 0.3417, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5243055820465088, + "rewards/margins": 1.5398471355438232, + "rewards/rejected": -3.064152956008911, + "step": 12660 + }, + { + "epoch": 0.83, + "learning_rate": 4.3247510013932377e-07, + "logits/chosen": -0.8389037251472473, + "logits/rejected": -0.9085738062858582, + "logps/chosen": -431.64239501953125, + "logps/rejected": -547.0601806640625, + "loss": 0.5185, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7945387363433838, + "rewards/margins": 1.1559791564941406, + "rewards/rejected": -2.9505176544189453, + "step": 12670 + }, + { + "epoch": 0.83, + "learning_rate": 4.2927045665335594e-07, + "logits/chosen": -0.5805936455726624, + "logits/rejected": -0.7223195433616638, + "logps/chosen": -361.5916442871094, + "logps/rejected": -439.7505798339844, + "loss": 0.549, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.9671390056610107, + "rewards/margins": 0.8171416521072388, + "rewards/rejected": -2.784280300140381, + "step": 12680 + }, + { + "epoch": 0.83, + "learning_rate": 4.260766151070439e-07, + "logits/chosen": -0.8927919268608093, + "logits/rejected": -0.881097137928009, + "logps/chosen": -398.68975830078125, + "logps/rejected": -498.0592346191406, + "loss": 0.5202, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7425625324249268, + "rewards/margins": 1.106300711631775, + "rewards/rejected": -2.848863124847412, + "step": 12690 + }, + { + "epoch": 0.83, + "learning_rate": 4.228935921610308e-07, + "logits/chosen": -1.243019461631775, + "logits/rejected": -0.9916293025016785, + "logps/chosen": -446.41015625, + "logps/rejected": -461.8265686035156, + "loss": 0.5839, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8269145488739014, + "rewards/margins": 0.7946796417236328, + "rewards/rejected": -2.621594190597534, + "step": 12700 + }, + { + "epoch": 0.83, + "eval_logits/chosen": -1.1701284646987915, + "eval_logits/rejected": -1.0344866514205933, + "eval_logps/chosen": -393.4916687011719, + "eval_logps/rejected": -482.1238098144531, + "eval_loss": 0.4880444407463074, + "eval_rewards/accuracies": 0.6690000295639038, + "eval_rewards/chosen": -1.6148673295974731, + "eval_rewards/margins": 1.0902516841888428, + "eval_rewards/rejected": -2.7051191329956055, + "eval_runtime": 732.9848, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.364, + "step": 12700 + }, + { + "epoch": 0.83, + "learning_rate": 4.1972140441952246e-07, + "logits/chosen": -1.0796960592269897, + "logits/rejected": -1.1231991052627563, + "logps/chosen": -391.99188232421875, + "logps/rejected": -498.2669372558594, + "loss": 0.5049, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.5240862369537354, + "rewards/margins": 1.0426585674285889, + "rewards/rejected": -2.5667450428009033, + "step": 12710 + }, + { + "epoch": 0.83, + "learning_rate": 4.165600684302046e-07, + "logits/chosen": -1.2587755918502808, + "logits/rejected": -1.1907663345336914, + "logps/chosen": -297.47430419921875, + "logps/rejected": -418.65264892578125, + "loss": 0.3999, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.2169029712677002, + "rewards/margins": 1.1122041940689087, + "rewards/rejected": -2.3291072845458984, + "step": 12720 + }, + { + "epoch": 0.83, + "learning_rate": 4.13409600684154e-07, + "logits/chosen": -1.240088701248169, + "logits/rejected": -1.0318121910095215, + "logps/chosen": -382.499755859375, + "logps/rejected": -454.86083984375, + "loss": 0.4893, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.6748014688491821, + "rewards/margins": 1.0154438018798828, + "rewards/rejected": -2.6902451515197754, + "step": 12730 + }, + { + "epoch": 0.83, + "learning_rate": 4.102700176157548e-07, + "logits/chosen": -1.3106701374053955, + "logits/rejected": -1.010158896446228, + "logps/chosen": -507.3036193847656, + "logps/rejected": -505.44390869140625, + "loss": 0.5486, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.8254835605621338, + "rewards/margins": 0.9592690467834473, + "rewards/rejected": -2.784752368927002, + "step": 12740 + }, + { + "epoch": 0.83, + "learning_rate": 4.0714133560260884e-07, + "logits/chosen": -1.215022087097168, + "logits/rejected": -1.1796523332595825, + "logps/chosen": -440.1405334472656, + "logps/rejected": -468.4134216308594, + "loss": 0.5162, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8060743808746338, + "rewards/margins": 0.8650380969047546, + "rewards/rejected": -2.671112537384033, + "step": 12750 + }, + { + "epoch": 0.83, + "learning_rate": 4.0402357096545527e-07, + "logits/chosen": -0.9729182124137878, + "logits/rejected": -1.1191080808639526, + "logps/chosen": -415.92535400390625, + "logps/rejected": -502.9111328125, + "loss": 0.4751, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.6416374444961548, + "rewards/margins": 0.9715788960456848, + "rewards/rejected": -2.6132164001464844, + "step": 12760 + }, + { + "epoch": 0.84, + "learning_rate": 4.0091673996808025e-07, + "logits/chosen": -1.3912891149520874, + "logits/rejected": -1.1987309455871582, + "logps/chosen": -401.42315673828125, + "logps/rejected": -461.45721435546875, + "loss": 0.5461, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.043630361557007, + "rewards/margins": 0.8466132283210754, + "rewards/rejected": -2.8902435302734375, + "step": 12770 + }, + { + "epoch": 0.84, + "learning_rate": 3.9782085881723776e-07, + "logits/chosen": -1.142588496208191, + "logits/rejected": -1.0690782070159912, + "logps/chosen": -325.50653076171875, + "logps/rejected": -455.8252868652344, + "loss": 0.5445, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.5990889072418213, + "rewards/margins": 1.1835216283798218, + "rewards/rejected": -2.7826106548309326, + "step": 12780 + }, + { + "epoch": 0.84, + "learning_rate": 3.947359436625592e-07, + "logits/chosen": -1.1046050786972046, + "logits/rejected": -0.9014351963996887, + "logps/chosen": -420.23046875, + "logps/rejected": -497.6240234375, + "loss": 0.4034, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8556751012802124, + "rewards/margins": 1.1495764255523682, + "rewards/rejected": -3.005251407623291, + "step": 12790 + }, + { + "epoch": 0.84, + "learning_rate": 3.9166201059647386e-07, + "logits/chosen": -1.1169941425323486, + "logits/rejected": -1.1712825298309326, + "logps/chosen": -450.0757751464844, + "logps/rejected": -472.9488220214844, + "loss": 0.6596, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.8822559118270874, + "rewards/margins": 0.5980395078659058, + "rewards/rejected": -2.4802956581115723, + "step": 12800 + }, + { + "epoch": 0.84, + "eval_logits/chosen": -1.1303144693374634, + "eval_logits/rejected": -0.9964880347251892, + "eval_logps/chosen": -411.1650390625, + "eval_logps/rejected": -499.86004638671875, + "eval_loss": 0.4863698184490204, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -1.7916008234024048, + "eval_rewards/margins": 1.0908806324005127, + "eval_rewards/rejected": -2.882481336593628, + "eval_runtime": 730.2818, + "eval_samples_per_second": 2.739, + "eval_steps_per_second": 1.369, + "step": 12800 + }, + { + "epoch": 0.84, + "learning_rate": 3.8859907565412194e-07, + "logits/chosen": -1.0224831104278564, + "logits/rejected": -1.3116912841796875, + "logps/chosen": -375.24005126953125, + "logps/rejected": -483.36016845703125, + "loss": 0.4366, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.955908179283142, + "rewards/margins": 1.0869848728179932, + "rewards/rejected": -3.042893171310425, + "step": 12810 + }, + { + "epoch": 0.84, + "learning_rate": 3.8554715481327303e-07, + "logits/chosen": -1.0355466604232788, + "logits/rejected": -1.0103200674057007, + "logps/chosen": -429.4497985839844, + "logps/rejected": -506.0899353027344, + "loss": 0.5426, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.9619081020355225, + "rewards/margins": 1.1203787326812744, + "rewards/rejected": -3.082286834716797, + "step": 12820 + }, + { + "epoch": 0.84, + "learning_rate": 3.8250626399424007e-07, + "logits/chosen": -1.2228209972381592, + "logits/rejected": -1.1389085054397583, + "logps/chosen": -451.1490173339844, + "logps/rejected": -517.5762939453125, + "loss": 0.5481, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.996708631515503, + "rewards/margins": 0.8527741432189941, + "rewards/rejected": -2.849483013153076, + "step": 12830 + }, + { + "epoch": 0.84, + "learning_rate": 3.7947641905980104e-07, + "logits/chosen": -1.0452697277069092, + "logits/rejected": -1.1376216411590576, + "logps/chosen": -362.114501953125, + "logps/rejected": -431.8590393066406, + "loss": 0.4584, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.5479605197906494, + "rewards/margins": 1.044252634048462, + "rewards/rejected": -2.5922131538391113, + "step": 12840 + }, + { + "epoch": 0.84, + "learning_rate": 3.764576358151098e-07, + "logits/chosen": -1.148471474647522, + "logits/rejected": -1.061302900314331, + "logps/chosen": -334.8290710449219, + "logps/rejected": -426.7361755371094, + "loss": 0.5224, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.6001205444335938, + "rewards/margins": 1.037514090538025, + "rewards/rejected": -2.637634754180908, + "step": 12850 + }, + { + "epoch": 0.84, + "learning_rate": 3.7344993000761944e-07, + "logits/chosen": -1.1538784503936768, + "logits/rejected": -1.0829343795776367, + "logps/chosen": -366.9634094238281, + "logps/rejected": -522.6871337890625, + "loss": 0.467, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.896482229232788, + "rewards/margins": 1.1275200843811035, + "rewards/rejected": -3.0240025520324707, + "step": 12860 + }, + { + "epoch": 0.84, + "learning_rate": 3.7045331732699585e-07, + "logits/chosen": -1.1157572269439697, + "logits/rejected": -0.9886964559555054, + "logps/chosen": -388.4722595214844, + "logps/rejected": -488.2107849121094, + "loss": 0.4774, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8327420949935913, + "rewards/margins": 1.3543800115585327, + "rewards/rejected": -3.187121868133545, + "step": 12870 + }, + { + "epoch": 0.84, + "learning_rate": 3.6746781340503993e-07, + "logits/chosen": -1.1017290353775024, + "logits/rejected": -0.8482405543327332, + "logps/chosen": -395.7943420410156, + "logps/rejected": -491.15899658203125, + "loss": 0.5106, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.613785743713379, + "rewards/margins": 1.0339164733886719, + "rewards/rejected": -2.647702217102051, + "step": 12880 + }, + { + "epoch": 0.84, + "learning_rate": 3.6449343381560116e-07, + "logits/chosen": -0.9443025588989258, + "logits/rejected": -0.782778799533844, + "logps/chosen": -458.7662048339844, + "logps/rejected": -560.9354248046875, + "loss": 0.4543, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.2734053134918213, + "rewards/margins": 1.155586838722229, + "rewards/rejected": -3.428992509841919, + "step": 12890 + }, + { + "epoch": 0.84, + "learning_rate": 3.615301940745017e-07, + "logits/chosen": -1.4761455059051514, + "logits/rejected": -0.9457404017448425, + "logps/chosen": -489.544189453125, + "logps/rejected": -482.9058532714844, + "loss": 0.5277, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7234907150268555, + "rewards/margins": 0.9853858947753906, + "rewards/rejected": -2.708876848220825, + "step": 12900 + }, + { + "epoch": 0.84, + "eval_logits/chosen": -1.1100314855575562, + "eval_logits/rejected": -0.9771387577056885, + "eval_logps/chosen": -417.5810241699219, + "eval_logps/rejected": -506.6070251464844, + "eval_loss": 0.4858885109424591, + "eval_rewards/accuracies": 0.6694999933242798, + "eval_rewards/chosen": -1.8557612895965576, + "eval_rewards/margins": 1.0941897630691528, + "eval_rewards/rejected": -2.949951171875, + "eval_runtime": 735.1341, + "eval_samples_per_second": 2.721, + "eval_steps_per_second": 1.36, + "step": 12900 + }, + { + "epoch": 0.84, + "learning_rate": 3.5857810963945084e-07, + "logits/chosen": -0.7660588622093201, + "logits/rejected": -0.8179569244384766, + "logps/chosen": -419.36907958984375, + "logps/rejected": -510.74688720703125, + "loss": 0.4735, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0485999584198, + "rewards/margins": 1.0620059967041016, + "rewards/rejected": -3.1106061935424805, + "step": 12910 + }, + { + "epoch": 0.85, + "learning_rate": 3.556371959099678e-07, + "logits/chosen": -1.3508169651031494, + "logits/rejected": -1.0648918151855469, + "logps/chosen": -485.3834533691406, + "logps/rejected": -553.64501953125, + "loss": 0.4721, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8820909261703491, + "rewards/margins": 1.0823203325271606, + "rewards/rejected": -2.9644112586975098, + "step": 12920 + }, + { + "epoch": 0.85, + "learning_rate": 3.5270746822729797e-07, + "logits/chosen": -1.164005160331726, + "logits/rejected": -0.9906147122383118, + "logps/chosen": -466.7239685058594, + "logps/rejected": -584.4600830078125, + "loss": 0.5559, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1924190521240234, + "rewards/margins": 1.0516093969345093, + "rewards/rejected": -3.244028091430664, + "step": 12930 + }, + { + "epoch": 0.85, + "learning_rate": 3.4978894187433746e-07, + "logits/chosen": -1.0629559755325317, + "logits/rejected": -1.0066983699798584, + "logps/chosen": -319.3057556152344, + "logps/rejected": -384.53765869140625, + "loss": 0.5708, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.6823055744171143, + "rewards/margins": 0.7614030241966248, + "rewards/rejected": -2.443708896636963, + "step": 12940 + }, + { + "epoch": 0.85, + "learning_rate": 3.468816320755486e-07, + "logits/chosen": -0.8171752691268921, + "logits/rejected": -0.8537039756774902, + "logps/chosen": -386.3449401855469, + "logps/rejected": -427.5325622558594, + "loss": 0.5521, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.6241718530654907, + "rewards/margins": 0.8079736828804016, + "rewards/rejected": -2.432145357131958, + "step": 12950 + }, + { + "epoch": 0.85, + "learning_rate": 3.4398555399688336e-07, + "logits/chosen": -1.1284414529800415, + "logits/rejected": -0.9537376165390015, + "logps/chosen": -435.8028259277344, + "logps/rejected": -462.26953125, + "loss": 0.7083, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.244523286819458, + "rewards/margins": 0.471309095621109, + "rewards/rejected": -2.715832233428955, + "step": 12960 + }, + { + "epoch": 0.85, + "learning_rate": 3.411007227457047e-07, + "logits/chosen": -1.224440097808838, + "logits/rejected": -0.9128150939941406, + "logps/chosen": -418.45770263671875, + "logps/rejected": -505.29766845703125, + "loss": 0.459, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.7069644927978516, + "rewards/margins": 1.191146969795227, + "rewards/rejected": -2.898111343383789, + "step": 12970 + }, + { + "epoch": 0.85, + "learning_rate": 3.382271533707043e-07, + "logits/chosen": -0.9220924377441406, + "logits/rejected": -0.8552336692810059, + "logps/chosen": -370.10992431640625, + "logps/rejected": -422.84381103515625, + "loss": 0.5728, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.785047173500061, + "rewards/margins": 0.7561759948730469, + "rewards/rejected": -2.5412230491638184, + "step": 12980 + }, + { + "epoch": 0.85, + "learning_rate": 3.353648608618287e-07, + "logits/chosen": -1.0896251201629639, + "logits/rejected": -0.9746156930923462, + "logps/chosen": -324.8300476074219, + "logps/rejected": -415.791015625, + "loss": 0.4819, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.5695884227752686, + "rewards/margins": 1.0093162059783936, + "rewards/rejected": -2.578904628753662, + "step": 12990 + }, + { + "epoch": 0.85, + "learning_rate": 3.3251386015019676e-07, + "logits/chosen": -1.3970059156417847, + "logits/rejected": -1.1359691619873047, + "logps/chosen": -388.5389709472656, + "logps/rejected": -452.7955627441406, + "loss": 0.4608, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8577959537506104, + "rewards/margins": 0.980010986328125, + "rewards/rejected": -2.8378071784973145, + "step": 13000 + }, + { + "epoch": 0.85, + "eval_logits/chosen": -1.0929280519485474, + "eval_logits/rejected": -0.9613592028617859, + "eval_logps/chosen": -421.5428161621094, + "eval_logps/rejected": -508.9826965332031, + "eval_loss": 0.4859364330768585, + "eval_rewards/accuracies": 0.6735000014305115, + "eval_rewards/chosen": -1.8953787088394165, + "eval_rewards/margins": 1.0783287286758423, + "eval_rewards/rejected": -2.973707437515259, + "eval_runtime": 733.5256, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.363, + "step": 13000 + }, + { + "epoch": 0.85, + "learning_rate": 3.296741661080255e-07, + "logits/chosen": -1.226764440536499, + "logits/rejected": -1.0761001110076904, + "logps/chosen": -456.44622802734375, + "logps/rejected": -541.66796875, + "loss": 0.7879, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.233079195022583, + "rewards/margins": 0.917628288269043, + "rewards/rejected": -3.150707244873047, + "step": 13010 + }, + { + "epoch": 0.85, + "learning_rate": 3.2684579354854974e-07, + "logits/chosen": -1.285154104232788, + "logits/rejected": -1.1769609451293945, + "logps/chosen": -517.5054931640625, + "logps/rejected": -646.6309204101562, + "loss": 0.4975, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.2504684925079346, + "rewards/margins": 1.1886699199676514, + "rewards/rejected": -3.439138412475586, + "step": 13020 + }, + { + "epoch": 0.85, + "learning_rate": 3.2402875722594653e-07, + "logits/chosen": -1.0246667861938477, + "logits/rejected": -1.128968596458435, + "logps/chosen": -337.71929931640625, + "logps/rejected": -448.1748962402344, + "loss": 0.526, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.68673837184906, + "rewards/margins": 1.0430407524108887, + "rewards/rejected": -2.729779005050659, + "step": 13030 + }, + { + "epoch": 0.85, + "learning_rate": 3.212230718352566e-07, + "logits/chosen": -1.0177228450775146, + "logits/rejected": -0.9467980265617371, + "logps/chosen": -415.2057189941406, + "logps/rejected": -393.44915771484375, + "loss": 0.661, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.90155827999115, + "rewards/margins": 0.41940444707870483, + "rewards/rejected": -2.32096266746521, + "step": 13040 + }, + { + "epoch": 0.85, + "learning_rate": 3.1842875201231025e-07, + "logits/chosen": -1.015235185623169, + "logits/rejected": -0.871992290019989, + "logps/chosen": -386.43499755859375, + "logps/rejected": -459.2357482910156, + "loss": 0.4994, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.6469495296478271, + "rewards/margins": 0.985759437084198, + "rewards/rejected": -2.63270902633667, + "step": 13050 + }, + { + "epoch": 0.85, + "learning_rate": 3.156458123336478e-07, + "logits/chosen": -0.8194257020950317, + "logits/rejected": -0.7359941601753235, + "logps/chosen": -321.8291015625, + "logps/rejected": -450.9248046875, + "loss": 0.424, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6331720352172852, + "rewards/margins": 1.3924449682235718, + "rewards/rejected": -3.0256171226501465, + "step": 13060 + }, + { + "epoch": 0.86, + "learning_rate": 3.128742673164459e-07, + "logits/chosen": -1.3152307271957397, + "logits/rejected": -0.8516858816146851, + "logps/chosen": -475.12158203125, + "logps/rejected": -545.9874267578125, + "loss": 0.4057, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.9219434261322021, + "rewards/margins": 1.1538183689117432, + "rewards/rejected": -3.0757617950439453, + "step": 13070 + }, + { + "epoch": 0.86, + "learning_rate": 3.101141314184414e-07, + "logits/chosen": -1.448878288269043, + "logits/rejected": -1.2280422449111938, + "logps/chosen": -382.58502197265625, + "logps/rejected": -466.08428955078125, + "loss": 0.4464, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.77200448513031, + "rewards/margins": 0.9271053075790405, + "rewards/rejected": -2.6991097927093506, + "step": 13080 + }, + { + "epoch": 0.86, + "learning_rate": 3.0736541903785526e-07, + "logits/chosen": -0.8780330419540405, + "logits/rejected": -0.9904329180717468, + "logps/chosen": -415.8080139160156, + "logps/rejected": -569.5153198242188, + "loss": 0.523, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.0679306983947754, + "rewards/margins": 1.0424773693084717, + "rewards/rejected": -3.110407590866089, + "step": 13090 + }, + { + "epoch": 0.86, + "learning_rate": 3.0462814451331704e-07, + "logits/chosen": -1.0929917097091675, + "logits/rejected": -0.8251727223396301, + "logps/chosen": -455.87774658203125, + "logps/rejected": -526.8124389648438, + "loss": 0.5661, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2520129680633545, + "rewards/margins": 0.710297167301178, + "rewards/rejected": -2.962310314178467, + "step": 13100 + }, + { + "epoch": 0.86, + "eval_logits/chosen": -1.0824329853057861, + "eval_logits/rejected": -0.9514362215995789, + "eval_logps/chosen": -421.4238586425781, + "eval_logps/rejected": -507.9122009277344, + "eval_loss": 0.48598480224609375, + "eval_rewards/accuracies": 0.6725000143051147, + "eval_rewards/chosen": -1.894188642501831, + "eval_rewards/margins": 1.0688139200210571, + "eval_rewards/rejected": -2.9630024433135986, + "eval_runtime": 733.4557, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.363, + "step": 13100 + }, + { + "epoch": 0.86, + "learning_rate": 3.019023221237927e-07, + "logits/chosen": -1.0396716594696045, + "logits/rejected": -0.8928040266036987, + "logps/chosen": -417.76763916015625, + "logps/rejected": -469.63482666015625, + "loss": 0.4552, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8103529214859009, + "rewards/margins": 1.0943458080291748, + "rewards/rejected": -2.9046988487243652, + "step": 13110 + }, + { + "epoch": 0.86, + "learning_rate": 2.991879660885058e-07, + "logits/chosen": -1.2439924478530884, + "logits/rejected": -1.165687084197998, + "logps/chosen": -440.7635803222656, + "logps/rejected": -545.8884887695312, + "loss": 0.475, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7721668481826782, + "rewards/margins": 1.2148511409759521, + "rewards/rejected": -2.987017869949341, + "step": 13120 + }, + { + "epoch": 0.86, + "learning_rate": 2.9648509056686786e-07, + "logits/chosen": -1.1436467170715332, + "logits/rejected": -0.9742773175239563, + "logps/chosen": -370.8565368652344, + "logps/rejected": -660.5780029296875, + "loss": 0.5094, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.939147710800171, + "rewards/margins": 3.1529717445373535, + "rewards/rejected": -5.0921196937561035, + "step": 13130 + }, + { + "epoch": 0.86, + "learning_rate": 2.937937096584012e-07, + "logits/chosen": -1.150695562362671, + "logits/rejected": -0.8165030479431152, + "logps/chosen": -500.3816833496094, + "logps/rejected": -528.1851806640625, + "loss": 0.5713, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0998072624206543, + "rewards/margins": 0.8828747868537903, + "rewards/rejected": -2.9826817512512207, + "step": 13140 + }, + { + "epoch": 0.86, + "learning_rate": 2.9111383740266756e-07, + "logits/chosen": -0.9092572927474976, + "logits/rejected": -0.9110123515129089, + "logps/chosen": -444.96563720703125, + "logps/rejected": -513.8148193359375, + "loss": 0.5191, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.0918147563934326, + "rewards/margins": 0.7463721036911011, + "rewards/rejected": -2.8381869792938232, + "step": 13150 + }, + { + "epoch": 0.86, + "learning_rate": 2.8844548777919255e-07, + "logits/chosen": -1.1265146732330322, + "logits/rejected": -1.0310777425765991, + "logps/chosen": -376.72259521484375, + "logps/rejected": -427.83428955078125, + "loss": 0.6115, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.728387475013733, + "rewards/margins": 0.7339189648628235, + "rewards/rejected": -2.462306261062622, + "step": 13160 + }, + { + "epoch": 0.86, + "learning_rate": 2.8578867470739594e-07, + "logits/chosen": -0.7835243940353394, + "logits/rejected": -0.7032335996627808, + "logps/chosen": -369.31378173828125, + "logps/rejected": -463.86767578125, + "loss": 0.4816, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8482542037963867, + "rewards/margins": 1.259765863418579, + "rewards/rejected": -3.1080198287963867, + "step": 13170 + }, + { + "epoch": 0.86, + "learning_rate": 2.8314341204651484e-07, + "logits/chosen": -1.4030210971832275, + "logits/rejected": -1.210394263267517, + "logps/chosen": -436.91845703125, + "logps/rejected": -492.1163635253906, + "loss": 0.3838, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.6087934970855713, + "rewards/margins": 1.2798042297363281, + "rewards/rejected": -2.8885979652404785, + "step": 13180 + }, + { + "epoch": 0.86, + "learning_rate": 2.805097135955362e-07, + "logits/chosen": -1.0102050304412842, + "logits/rejected": -0.8716405630111694, + "logps/chosen": -389.81170654296875, + "logps/rejected": -457.72113037109375, + "loss": 0.4974, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.7728769779205322, + "rewards/margins": 1.0283530950546265, + "rewards/rejected": -2.801229953765869, + "step": 13190 + }, + { + "epoch": 0.86, + "learning_rate": 2.778875930931213e-07, + "logits/chosen": -1.0638980865478516, + "logits/rejected": -0.816801905632019, + "logps/chosen": -417.91619873046875, + "logps/rejected": -522.2945556640625, + "loss": 0.4732, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8279731273651123, + "rewards/margins": 1.217405080795288, + "rewards/rejected": -3.0453782081604004, + "step": 13200 + }, + { + "epoch": 0.86, + "eval_logits/chosen": -1.0934062004089355, + "eval_logits/rejected": -0.9613891243934631, + "eval_logps/chosen": -416.2484130859375, + "eval_logps/rejected": -504.401611328125, + "eval_loss": 0.4857296645641327, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -1.8424345254898071, + "eval_rewards/margins": 1.0854624509811401, + "eval_rewards/rejected": -2.9278969764709473, + "eval_runtime": 731.1175, + "eval_samples_per_second": 2.736, + "eval_steps_per_second": 1.368, + "step": 13200 + }, + { + "epoch": 0.86, + "learning_rate": 2.7527706421753426e-07, + "logits/chosen": -1.2587956190109253, + "logits/rejected": -1.0719627141952515, + "logps/chosen": -374.5654602050781, + "logps/rejected": -472.62548828125, + "loss": 0.5174, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.7616428136825562, + "rewards/margins": 0.9357126355171204, + "rewards/rejected": -2.6973557472229004, + "step": 13210 + }, + { + "epoch": 0.86, + "learning_rate": 2.726781405865736e-07, + "logits/chosen": -1.1134307384490967, + "logits/rejected": -0.9242690205574036, + "logps/chosen": -492.21295166015625, + "logps/rejected": -477.84063720703125, + "loss": 0.5215, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8954721689224243, + "rewards/margins": 1.0139615535736084, + "rewards/rejected": -2.9094338417053223, + "step": 13220 + }, + { + "epoch": 0.87, + "learning_rate": 2.7009083575749687e-07, + "logits/chosen": -0.9584072828292847, + "logits/rejected": -0.9138525724411011, + "logps/chosen": -425.8014221191406, + "logps/rejected": -512.4229125976562, + "loss": 0.5108, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.819411039352417, + "rewards/margins": 0.8825702667236328, + "rewards/rejected": -2.7019810676574707, + "step": 13230 + }, + { + "epoch": 0.87, + "learning_rate": 2.6751516322695457e-07, + "logits/chosen": -1.102455496788025, + "logits/rejected": -1.0879199504852295, + "logps/chosen": -396.62811279296875, + "logps/rejected": -453.2647399902344, + "loss": 0.5622, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.063845634460449, + "rewards/margins": 0.6267648935317993, + "rewards/rejected": -2.690610885620117, + "step": 13240 + }, + { + "epoch": 0.87, + "learning_rate": 2.649511364309154e-07, + "logits/chosen": -1.3207348585128784, + "logits/rejected": -1.175112009048462, + "logps/chosen": -387.6021423339844, + "logps/rejected": -464.5248107910156, + "loss": 0.4696, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.8531246185302734, + "rewards/margins": 0.9780380129814148, + "rewards/rejected": -2.831162214279175, + "step": 13250 + }, + { + "epoch": 0.87, + "learning_rate": 2.6239876874460003e-07, + "logits/chosen": -1.285677194595337, + "logits/rejected": -1.268032431602478, + "logps/chosen": -476.5404357910156, + "logps/rejected": -601.5635986328125, + "loss": 0.4047, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.921252965927124, + "rewards/margins": 1.5557920932769775, + "rewards/rejected": -3.4770455360412598, + "step": 13260 + }, + { + "epoch": 0.87, + "learning_rate": 2.5985807348240744e-07, + "logits/chosen": -1.1611740589141846, + "logits/rejected": -0.7273094058036804, + "logps/chosen": -423.51837158203125, + "logps/rejected": -547.768798828125, + "loss": 0.3697, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.911747694015503, + "rewards/margins": 1.7219756841659546, + "rewards/rejected": -3.633723020553589, + "step": 13270 + }, + { + "epoch": 0.87, + "learning_rate": 2.5732906389785014e-07, + "logits/chosen": -1.2326469421386719, + "logits/rejected": -1.1700255870819092, + "logps/chosen": -460.8599548339844, + "logps/rejected": -551.9319458007812, + "loss": 0.4286, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.831538438796997, + "rewards/margins": 1.2921754121780396, + "rewards/rejected": -3.123713970184326, + "step": 13280 + }, + { + "epoch": 0.87, + "learning_rate": 2.5481175318347956e-07, + "logits/chosen": -0.9984075427055359, + "logits/rejected": -1.1866605281829834, + "logps/chosen": -407.84014892578125, + "logps/rejected": -542.4286499023438, + "loss": 0.458, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8034251928329468, + "rewards/margins": 1.1669247150421143, + "rewards/rejected": -2.9703497886657715, + "step": 13290 + }, + { + "epoch": 0.87, + "learning_rate": 2.5230615447082246e-07, + "logits/chosen": -1.0014846324920654, + "logits/rejected": -0.8525576591491699, + "logps/chosen": -443.6893005371094, + "logps/rejected": -539.8489990234375, + "loss": 0.5427, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.9237266778945923, + "rewards/margins": 1.1094491481781006, + "rewards/rejected": -3.0331757068634033, + "step": 13300 + }, + { + "epoch": 0.87, + "eval_logits/chosen": -1.0766102075576782, + "eval_logits/rejected": -0.9451166391372681, + "eval_logps/chosen": -422.79327392578125, + "eval_logps/rejected": -511.8057861328125, + "eval_loss": 0.4857572019100189, + "eval_rewards/accuracies": 0.6710000038146973, + "eval_rewards/chosen": -1.9078832864761353, + "eval_rewards/margins": 1.09405517578125, + "eval_rewards/rejected": -3.0019383430480957, + "eval_runtime": 732.2689, + "eval_samples_per_second": 2.731, + "eval_steps_per_second": 1.366, + "step": 13300 + }, + { + "epoch": 0.87, + "learning_rate": 2.49812280830308e-07, + "logits/chosen": -1.0418148040771484, + "logits/rejected": -0.788641095161438, + "logps/chosen": -402.3210144042969, + "logps/rejected": -550.3643798828125, + "loss": 0.3707, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.7728382349014282, + "rewards/margins": 1.828993797302246, + "rewards/rejected": -3.6018319129943848, + "step": 13310 + }, + { + "epoch": 0.87, + "learning_rate": 2.4733014527120457e-07, + "logits/chosen": -0.8111721277236938, + "logits/rejected": -0.937555193901062, + "logps/chosen": -460.4014587402344, + "logps/rejected": -551.224853515625, + "loss": 0.5392, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.5798659324645996, + "rewards/margins": 1.1371663808822632, + "rewards/rejected": -3.7170321941375732, + "step": 13320 + }, + { + "epoch": 0.87, + "learning_rate": 2.4485976074154565e-07, + "logits/chosen": -1.1172932386398315, + "logits/rejected": -1.1888206005096436, + "logps/chosen": -412.2958068847656, + "logps/rejected": -470.85528564453125, + "loss": 0.6788, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.032978057861328, + "rewards/margins": 0.285163015127182, + "rewards/rejected": -2.318140983581543, + "step": 13330 + }, + { + "epoch": 0.87, + "learning_rate": 2.4240114012806763e-07, + "logits/chosen": -1.0329598188400269, + "logits/rejected": -1.0528901815414429, + "logps/chosen": -394.06512451171875, + "logps/rejected": -464.07159423828125, + "loss": 0.5461, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8626697063446045, + "rewards/margins": 0.8707631826400757, + "rewards/rejected": -2.7334327697753906, + "step": 13340 + }, + { + "epoch": 0.87, + "learning_rate": 2.399542962561399e-07, + "logits/chosen": -1.0653612613677979, + "logits/rejected": -0.863553524017334, + "logps/chosen": -392.567138671875, + "logps/rejected": -456.32061767578125, + "loss": 0.4809, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6711117029190063, + "rewards/margins": 1.1116281747817993, + "rewards/rejected": -2.7827396392822266, + "step": 13350 + }, + { + "epoch": 0.87, + "learning_rate": 2.3751924188969876e-07, + "logits/chosen": -1.0322988033294678, + "logits/rejected": -0.889003574848175, + "logps/chosen": -443.11468505859375, + "logps/rejected": -542.2542114257812, + "loss": 0.484, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.913761854171753, + "rewards/margins": 1.1897408962249756, + "rewards/rejected": -3.1035027503967285, + "step": 13360 + }, + { + "epoch": 0.87, + "learning_rate": 2.3509598973118024e-07, + "logits/chosen": -1.3392784595489502, + "logits/rejected": -1.1734321117401123, + "logps/chosen": -397.5035095214844, + "logps/rejected": -399.9672546386719, + "loss": 0.6019, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7649815082550049, + "rewards/margins": 0.652515709400177, + "rewards/rejected": -2.417497158050537, + "step": 13370 + }, + { + "epoch": 0.88, + "learning_rate": 2.326845524214555e-07, + "logits/chosen": -0.9857600927352905, + "logits/rejected": -1.0095943212509155, + "logps/chosen": -435.65728759765625, + "logps/rejected": -429.5391540527344, + "loss": 0.73, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.0060293674468994, + "rewards/margins": 0.20006613433361053, + "rewards/rejected": -2.2060954570770264, + "step": 13380 + }, + { + "epoch": 0.88, + "learning_rate": 2.3028494253976158e-07, + "logits/chosen": -1.102888822555542, + "logits/rejected": -0.930350124835968, + "logps/chosen": -558.3133544921875, + "logps/rejected": -571.853515625, + "loss": 0.5706, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1704790592193604, + "rewards/margins": 0.7812148928642273, + "rewards/rejected": -2.9516937732696533, + "step": 13390 + }, + { + "epoch": 0.88, + "learning_rate": 2.2789717260364026e-07, + "logits/chosen": -1.1375192403793335, + "logits/rejected": -0.9567796587944031, + "logps/chosen": -323.5530700683594, + "logps/rejected": -386.8634338378906, + "loss": 0.5223, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.5844700336456299, + "rewards/margins": 0.8503804206848145, + "rewards/rejected": -2.4348502159118652, + "step": 13400 + }, + { + "epoch": 0.88, + "eval_logits/chosen": -1.087154746055603, + "eval_logits/rejected": -0.9559369087219238, + "eval_logps/chosen": -422.0847473144531, + "eval_logps/rejected": -508.4212951660156, + "eval_loss": 0.4862857758998871, + "eval_rewards/accuracies": 0.671999990940094, + "eval_rewards/chosen": -1.9007982015609741, + "eval_rewards/margins": 1.0672953128814697, + "eval_rewards/rejected": -2.968093156814575, + "eval_runtime": 733.4672, + "eval_samples_per_second": 2.727, + "eval_steps_per_second": 1.363, + "step": 13400 + }, + { + "epoch": 0.88, + "learning_rate": 2.255212550688682e-07, + "logits/chosen": -1.1616137027740479, + "logits/rejected": -1.485405683517456, + "logps/chosen": -426.1238708496094, + "logps/rejected": -621.4586791992188, + "loss": 0.4524, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.081688165664673, + "rewards/margins": 1.3114808797836304, + "rewards/rejected": -3.393169403076172, + "step": 13410 + }, + { + "epoch": 0.88, + "learning_rate": 2.2315720232939598e-07, + "logits/chosen": -1.5970458984375, + "logits/rejected": -1.1075289249420166, + "logps/chosen": -440.81884765625, + "logps/rejected": -485.1519470214844, + "loss": 0.3706, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.7870508432388306, + "rewards/margins": 1.339354395866394, + "rewards/rejected": -3.1264052391052246, + "step": 13420 + }, + { + "epoch": 0.88, + "learning_rate": 2.2080502671727956e-07, + "logits/chosen": -1.2229093313217163, + "logits/rejected": -1.0283030271530151, + "logps/chosen": -372.5098876953125, + "logps/rejected": -454.7687072753906, + "loss": 0.4706, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.524280309677124, + "rewards/margins": 1.04086434841156, + "rewards/rejected": -2.5651450157165527, + "step": 13430 + }, + { + "epoch": 0.88, + "learning_rate": 2.1846474050262078e-07, + "logits/chosen": -1.0370843410491943, + "logits/rejected": -0.8305977582931519, + "logps/chosen": -425.2176208496094, + "logps/rejected": -456.01605224609375, + "loss": 0.4756, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7865688800811768, + "rewards/margins": 0.9659556150436401, + "rewards/rejected": -2.7525246143341064, + "step": 13440 + }, + { + "epoch": 0.88, + "learning_rate": 2.1613635589349756e-07, + "logits/chosen": -0.8326600193977356, + "logits/rejected": -0.7741021513938904, + "logps/chosen": -381.1373291015625, + "logps/rejected": -534.6483154296875, + "loss": 0.4565, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8397401571273804, + "rewards/margins": 1.259902834892273, + "rewards/rejected": -3.0996429920196533, + "step": 13450 + }, + { + "epoch": 0.88, + "learning_rate": 2.1381988503590578e-07, + "logits/chosen": -0.7525622248649597, + "logits/rejected": -0.8662757873535156, + "logps/chosen": -411.94189453125, + "logps/rejected": -541.89892578125, + "loss": 0.3546, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.9607007503509521, + "rewards/margins": 1.361067771911621, + "rewards/rejected": -3.3217685222625732, + "step": 13460 + }, + { + "epoch": 0.88, + "learning_rate": 2.11515340013691e-07, + "logits/chosen": -1.2946085929870605, + "logits/rejected": -1.1581755876541138, + "logps/chosen": -438.44049072265625, + "logps/rejected": -542.212158203125, + "loss": 0.4053, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.072679042816162, + "rewards/margins": 1.253519058227539, + "rewards/rejected": -3.326197862625122, + "step": 13470 + }, + { + "epoch": 0.88, + "learning_rate": 2.092227328484897e-07, + "logits/chosen": -0.8586756587028503, + "logits/rejected": -0.8986477851867676, + "logps/chosen": -372.24542236328125, + "logps/rejected": -510.7012634277344, + "loss": 0.5042, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.6823673248291016, + "rewards/margins": 1.0779831409454346, + "rewards/rejected": -2.760350465774536, + "step": 13480 + }, + { + "epoch": 0.88, + "learning_rate": 2.0694207549966345e-07, + "logits/chosen": -0.8356062173843384, + "logits/rejected": -0.9943629503250122, + "logps/chosen": -416.49627685546875, + "logps/rejected": -452.17205810546875, + "loss": 0.6632, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.0758779048919678, + "rewards/margins": 0.525452733039856, + "rewards/rejected": -2.601330280303955, + "step": 13490 + }, + { + "epoch": 0.88, + "learning_rate": 2.0467337986423864e-07, + "logits/chosen": -1.269989252090454, + "logits/rejected": -1.0888997316360474, + "logps/chosen": -505.70941162109375, + "logps/rejected": -569.6568603515625, + "loss": 0.4808, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9998592138290405, + "rewards/margins": 0.971601665019989, + "rewards/rejected": -2.9714608192443848, + "step": 13500 + }, + { + "epoch": 0.88, + "eval_logits/chosen": -1.0681487321853638, + "eval_logits/rejected": -0.9375593066215515, + "eval_logps/chosen": -425.8811950683594, + "eval_logps/rejected": -514.4193115234375, + "eval_loss": 0.4859347641468048, + "eval_rewards/accuracies": 0.6735000014305115, + "eval_rewards/chosen": -1.9387621879577637, + "eval_rewards/margins": 1.0893113613128662, + "eval_rewards/rejected": -3.028073787689209, + "eval_runtime": 730.6979, + "eval_samples_per_second": 2.737, + "eval_steps_per_second": 1.369, + "step": 13500 + }, + { + "epoch": 0.88, + "learning_rate": 2.0241665777684272e-07, + "logits/chosen": -1.312229871749878, + "logits/rejected": -1.1031113862991333, + "logps/chosen": -432.3946228027344, + "logps/rejected": -561.6215209960938, + "loss": 0.3229, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.6410160064697266, + "rewards/margins": 1.7284218072891235, + "rewards/rejected": -3.3694376945495605, + "step": 13510 + }, + { + "epoch": 0.88, + "learning_rate": 2.0017192100964366e-07, + "logits/chosen": -0.85126131772995, + "logits/rejected": -0.8515372276306152, + "logps/chosen": -403.20086669921875, + "logps/rejected": -506.77337646484375, + "loss": 0.4693, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0324487686157227, + "rewards/margins": 1.0145536661148071, + "rewards/rejected": -3.0470023155212402, + "step": 13520 + }, + { + "epoch": 0.89, + "learning_rate": 1.9793918127228777e-07, + "logits/chosen": -1.3386536836624146, + "logits/rejected": -0.9312573671340942, + "logps/chosen": -534.556640625, + "logps/rejected": -611.8395385742188, + "loss": 0.4614, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.1825718879699707, + "rewards/margins": 1.3136504888534546, + "rewards/rejected": -3.4962220191955566, + "step": 13530 + }, + { + "epoch": 0.89, + "learning_rate": 1.9571845021184005e-07, + "logits/chosen": -0.8405078053474426, + "logits/rejected": -0.8910934329032898, + "logps/chosen": -434.1742248535156, + "logps/rejected": -528.9740600585938, + "loss": 0.5757, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0318832397460938, + "rewards/margins": 0.9068182706832886, + "rewards/rejected": -2.938701629638672, + "step": 13540 + }, + { + "epoch": 0.89, + "learning_rate": 1.9350973941272027e-07, + "logits/chosen": -1.212944746017456, + "logits/rejected": -0.9825285077095032, + "logps/chosen": -435.71917724609375, + "logps/rejected": -512.9835205078125, + "loss": 0.5183, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.319441318511963, + "rewards/margins": 0.9776519536972046, + "rewards/rejected": -3.297093152999878, + "step": 13550 + }, + { + "epoch": 0.89, + "learning_rate": 1.9131306039664676e-07, + "logits/chosen": -0.9572928547859192, + "logits/rejected": -0.8474523425102234, + "logps/chosen": -381.08544921875, + "logps/rejected": -526.7711791992188, + "loss": 0.4356, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.8118137121200562, + "rewards/margins": 1.213293433189392, + "rewards/rejected": -3.0251073837280273, + "step": 13560 + }, + { + "epoch": 0.89, + "learning_rate": 1.8912842462257358e-07, + "logits/chosen": -1.0587059259414673, + "logits/rejected": -0.9345768094062805, + "logps/chosen": -402.04144287109375, + "logps/rejected": -514.9813232421875, + "loss": 0.4445, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8187220096588135, + "rewards/margins": 1.2992680072784424, + "rewards/rejected": -3.117990016937256, + "step": 13570 + }, + { + "epoch": 0.89, + "learning_rate": 1.869558434866303e-07, + "logits/chosen": -1.0689226388931274, + "logits/rejected": -1.2264662981033325, + "logps/chosen": -382.54583740234375, + "logps/rejected": -523.0657348632812, + "loss": 0.5252, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.0145599842071533, + "rewards/margins": 1.2249000072479248, + "rewards/rejected": -3.239459991455078, + "step": 13580 + }, + { + "epoch": 0.89, + "learning_rate": 1.847953283220652e-07, + "logits/chosen": -1.1595171689987183, + "logits/rejected": -0.9138990640640259, + "logps/chosen": -453.76806640625, + "logps/rejected": -552.489990234375, + "loss": 0.3843, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.9765832424163818, + "rewards/margins": 1.7735812664031982, + "rewards/rejected": -3.750164747238159, + "step": 13590 + }, + { + "epoch": 0.89, + "learning_rate": 1.8264689039918265e-07, + "logits/chosen": -1.06974196434021, + "logits/rejected": -0.9910194277763367, + "logps/chosen": -487.11968994140625, + "logps/rejected": -554.9024658203125, + "loss": 0.5138, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.272892951965332, + "rewards/margins": 0.9520902633666992, + "rewards/rejected": -3.2249832153320312, + "step": 13600 + }, + { + "epoch": 0.89, + "eval_logits/chosen": -1.066765546798706, + "eval_logits/rejected": -0.9360716938972473, + "eval_logps/chosen": -430.43524169921875, + "eval_logps/rejected": -518.9196166992188, + "eval_loss": 0.48558491468429565, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -1.9843029975891113, + "eval_rewards/margins": 1.0887740850448608, + "eval_rewards/rejected": -3.0730769634246826, + "eval_runtime": 730.1997, + "eval_samples_per_second": 2.739, + "eval_steps_per_second": 1.369, + "step": 13600 + }, + { + "epoch": 0.89, + "learning_rate": 1.8051054092528857e-07, + "logits/chosen": -1.1157863140106201, + "logits/rejected": -0.9228916168212891, + "logps/chosen": -462.395263671875, + "logps/rejected": -571.1812744140625, + "loss": 0.4968, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.012938976287842, + "rewards/margins": 1.185462236404419, + "rewards/rejected": -3.1984007358551025, + "step": 13610 + }, + { + "epoch": 0.89, + "learning_rate": 1.783862910446271e-07, + "logits/chosen": -0.8563364148139954, + "logits/rejected": -0.8741576075553894, + "logps/chosen": -356.21136474609375, + "logps/rejected": -471.511474609375, + "loss": 0.4717, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8335803747177124, + "rewards/margins": 1.2461497783660889, + "rewards/rejected": -3.079730272293091, + "step": 13620 + }, + { + "epoch": 0.89, + "learning_rate": 1.762741518383271e-07, + "logits/chosen": -1.1820201873779297, + "logits/rejected": -0.908908486366272, + "logps/chosen": -425.9429626464844, + "logps/rejected": -497.426513671875, + "loss": 0.5404, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0411064624786377, + "rewards/margins": 0.9914758801460266, + "rewards/rejected": -3.0325820446014404, + "step": 13630 + }, + { + "epoch": 0.89, + "learning_rate": 1.7417413432434082e-07, + "logits/chosen": -1.0338904857635498, + "logits/rejected": -0.9325830340385437, + "logps/chosen": -451.5857849121094, + "logps/rejected": -513.577880859375, + "loss": 0.4921, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9892606735229492, + "rewards/margins": 1.1475944519042969, + "rewards/rejected": -3.136855363845825, + "step": 13640 + }, + { + "epoch": 0.89, + "learning_rate": 1.7208624945738855e-07, + "logits/chosen": -1.2731138467788696, + "logits/rejected": -1.2202355861663818, + "logps/chosen": -426.15582275390625, + "logps/rejected": -516.65283203125, + "loss": 0.5701, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0807149410247803, + "rewards/margins": 0.8305963277816772, + "rewards/rejected": -2.911311626434326, + "step": 13650 + }, + { + "epoch": 0.89, + "learning_rate": 1.7001050812889995e-07, + "logits/chosen": -1.3216979503631592, + "logits/rejected": -1.127280592918396, + "logps/chosen": -480.98974609375, + "logps/rejected": -547.7494506835938, + "loss": 0.5034, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2545692920684814, + "rewards/margins": 1.0701066255569458, + "rewards/rejected": -3.3246757984161377, + "step": 13660 + }, + { + "epoch": 0.89, + "learning_rate": 1.679469211669596e-07, + "logits/chosen": -1.0741422176361084, + "logits/rejected": -0.8863421678543091, + "logps/chosen": -439.17437744140625, + "logps/rejected": -493.56982421875, + "loss": 0.4553, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1422924995422363, + "rewards/margins": 1.0599985122680664, + "rewards/rejected": -3.2022907733917236, + "step": 13670 + }, + { + "epoch": 0.9, + "learning_rate": 1.6589549933624715e-07, + "logits/chosen": -1.0650371313095093, + "logits/rejected": -0.8889235258102417, + "logps/chosen": -416.11181640625, + "logps/rejected": -547.8284301757812, + "loss": 0.3695, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.7590478658676147, + "rewards/margins": 1.8320204019546509, + "rewards/rejected": -3.5910682678222656, + "step": 13680 + }, + { + "epoch": 0.9, + "learning_rate": 1.638562533379845e-07, + "logits/chosen": -0.9663646817207336, + "logits/rejected": -0.861709475517273, + "logps/chosen": -444.4697265625, + "logps/rejected": -477.4546813964844, + "loss": 0.5241, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.8876707553863525, + "rewards/margins": 0.9525297284126282, + "rewards/rejected": -2.840200424194336, + "step": 13690 + }, + { + "epoch": 0.9, + "learning_rate": 1.6182919380987676e-07, + "logits/chosen": -1.0882418155670166, + "logits/rejected": -1.0266286134719849, + "logps/chosen": -420.1298828125, + "logps/rejected": -474.15777587890625, + "loss": 0.5878, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.9441192150115967, + "rewards/margins": 0.6940501928329468, + "rewards/rejected": -2.638169765472412, + "step": 13700 + }, + { + "epoch": 0.9, + "eval_logits/chosen": -1.0581167936325073, + "eval_logits/rejected": -0.9280232191085815, + "eval_logps/chosen": -436.266357421875, + "eval_logps/rejected": -523.874267578125, + "eval_loss": 0.48549070954322815, + "eval_rewards/accuracies": 0.6694999933242798, + "eval_rewards/chosen": -2.042614221572876, + "eval_rewards/margins": 1.0800092220306396, + "eval_rewards/rejected": -3.1226232051849365, + "eval_runtime": 732.6369, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 13700 + }, + { + "epoch": 0.9, + "learning_rate": 1.598143313260603e-07, + "logits/chosen": -0.7987823486328125, + "logits/rejected": -0.8568712472915649, + "logps/chosen": -401.6100158691406, + "logps/rejected": -476.331298828125, + "loss": 0.5398, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.1355624198913574, + "rewards/margins": 0.8673583269119263, + "rewards/rejected": -3.0029211044311523, + "step": 13710 + }, + { + "epoch": 0.9, + "learning_rate": 1.5781167639704415e-07, + "logits/chosen": -1.1982240676879883, + "logits/rejected": -0.8309241533279419, + "logps/chosen": -548.0106811523438, + "logps/rejected": -538.3633422851562, + "loss": 0.4387, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1584393978118896, + "rewards/margins": 1.145960807800293, + "rewards/rejected": -3.3044002056121826, + "step": 13720 + }, + { + "epoch": 0.9, + "learning_rate": 1.5582123946965787e-07, + "logits/chosen": -0.946783185005188, + "logits/rejected": -0.7460772395133972, + "logps/chosen": -436.0660095214844, + "logps/rejected": -565.3934326171875, + "loss": 0.4354, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.016709566116333, + "rewards/margins": 1.191794991493225, + "rewards/rejected": -3.2085044384002686, + "step": 13730 + }, + { + "epoch": 0.9, + "learning_rate": 1.5384303092699504e-07, + "logits/chosen": -1.15846848487854, + "logits/rejected": -0.7557247281074524, + "logps/chosen": -466.0997009277344, + "logps/rejected": -632.2220458984375, + "loss": 0.3254, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.7911508083343506, + "rewards/margins": 1.678033471107483, + "rewards/rejected": -3.469184160232544, + "step": 13740 + }, + { + "epoch": 0.9, + "learning_rate": 1.518770610883613e-07, + "logits/chosen": -0.9040036201477051, + "logits/rejected": -0.802951991558075, + "logps/chosen": -455.0104064941406, + "logps/rejected": -563.3392333984375, + "loss": 0.3819, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -2.4047179222106934, + "rewards/margins": 1.3774263858795166, + "rewards/rejected": -3.782144069671631, + "step": 13750 + }, + { + "epoch": 0.9, + "learning_rate": 1.4992334020921735e-07, + "logits/chosen": -1.0886790752410889, + "logits/rejected": -1.0631605386734009, + "logps/chosen": -353.92230224609375, + "logps/rejected": -463.3160705566406, + "loss": 0.3949, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8094984292984009, + "rewards/margins": 1.3900763988494873, + "rewards/rejected": -3.1995749473571777, + "step": 13760 + }, + { + "epoch": 0.9, + "learning_rate": 1.4798187848112905e-07, + "logits/chosen": -1.0941932201385498, + "logits/rejected": -0.7571912407875061, + "logps/chosen": -474.4635314941406, + "logps/rejected": -545.6170654296875, + "loss": 0.5698, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.514927864074707, + "rewards/margins": 1.1201101541519165, + "rewards/rejected": -3.635037899017334, + "step": 13770 + }, + { + "epoch": 0.9, + "learning_rate": 1.460526860317113e-07, + "logits/chosen": -1.2194105386734009, + "logits/rejected": -1.0897470712661743, + "logps/chosen": -366.4600524902344, + "logps/rejected": -543.8425903320312, + "loss": 0.4088, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9317538738250732, + "rewards/margins": 1.4563566446304321, + "rewards/rejected": -3.388110637664795, + "step": 13780 + }, + { + "epoch": 0.9, + "learning_rate": 1.441357729245771e-07, + "logits/chosen": -1.2731847763061523, + "logits/rejected": -0.9451271891593933, + "logps/chosen": -504.37548828125, + "logps/rejected": -548.6998901367188, + "loss": 0.5107, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.5638363361358643, + "rewards/margins": 1.0031163692474365, + "rewards/rejected": -3.56695294380188, + "step": 13790 + }, + { + "epoch": 0.9, + "learning_rate": 1.4223114915928482e-07, + "logits/chosen": -0.6414790153503418, + "logits/rejected": -0.7103398442268372, + "logps/chosen": -432.17767333984375, + "logps/rejected": -554.3941650390625, + "loss": 0.4051, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0625510215759277, + "rewards/margins": 1.1544933319091797, + "rewards/rejected": -3.2170441150665283, + "step": 13800 + }, + { + "epoch": 0.9, + "eval_logits/chosen": -1.058732509613037, + "eval_logits/rejected": -0.9284118413925171, + "eval_logps/chosen": -435.3294982910156, + "eval_logps/rejected": -524.1821899414062, + "eval_loss": 0.4853297770023346, + "eval_rewards/accuracies": 0.6725000143051147, + "eval_rewards/chosen": -2.033245325088501, + "eval_rewards/margins": 1.0924570560455322, + "eval_rewards/rejected": -3.125702142715454, + "eval_runtime": 731.8825, + "eval_samples_per_second": 2.733, + "eval_steps_per_second": 1.366, + "step": 13800 + }, + { + "epoch": 0.9, + "learning_rate": 1.403388246712842e-07, + "logits/chosen": -1.068618893623352, + "logits/rejected": -0.9264750480651855, + "logps/chosen": -354.43719482421875, + "logps/rejected": -426.0357971191406, + "loss": 0.5102, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9018609523773193, + "rewards/margins": 0.7875036597251892, + "rewards/rejected": -2.6893649101257324, + "step": 13810 + }, + { + "epoch": 0.9, + "learning_rate": 1.3845880933186757e-07, + "logits/chosen": -1.1902129650115967, + "logits/rejected": -1.0742738246917725, + "logps/chosen": -435.77813720703125, + "logps/rejected": -479.9771423339844, + "loss": 0.556, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.9703929424285889, + "rewards/margins": 0.7843495607376099, + "rewards/rejected": -2.7547426223754883, + "step": 13820 + }, + { + "epoch": 0.9, + "learning_rate": 1.3659111294811457e-07, + "logits/chosen": -1.0660393238067627, + "logits/rejected": -1.0473939180374146, + "logps/chosen": -409.37060546875, + "logps/rejected": -486.2066955566406, + "loss": 0.5132, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.175274133682251, + "rewards/margins": 0.947323203086853, + "rewards/rejected": -3.1225974559783936, + "step": 13830 + }, + { + "epoch": 0.91, + "learning_rate": 1.347357452628459e-07, + "logits/chosen": -1.465331792831421, + "logits/rejected": -1.3321690559387207, + "logps/chosen": -442.4307556152344, + "logps/rejected": -534.9903564453125, + "loss": 0.5425, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.9713321924209595, + "rewards/margins": 1.0006132125854492, + "rewards/rejected": -2.971945285797119, + "step": 13840 + }, + { + "epoch": 0.91, + "learning_rate": 1.3289271595456732e-07, + "logits/chosen": -1.0778005123138428, + "logits/rejected": -0.7507137656211853, + "logps/chosen": -442.45904541015625, + "logps/rejected": -531.1512451171875, + "loss": 0.4908, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3812978267669678, + "rewards/margins": 1.144858717918396, + "rewards/rejected": -3.5261569023132324, + "step": 13850 + }, + { + "epoch": 0.91, + "learning_rate": 1.310620346374228e-07, + "logits/chosen": -1.010206937789917, + "logits/rejected": -0.8545116186141968, + "logps/chosen": -430.58880615234375, + "logps/rejected": -510.44842529296875, + "loss": 0.4736, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9836689233779907, + "rewards/margins": 1.1666280031204224, + "rewards/rejected": -3.150296688079834, + "step": 13860 + }, + { + "epoch": 0.91, + "learning_rate": 1.2924371086114274e-07, + "logits/chosen": -1.1741849184036255, + "logits/rejected": -0.7831594347953796, + "logps/chosen": -452.38055419921875, + "logps/rejected": -548.46728515625, + "loss": 0.5199, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.1631908416748047, + "rewards/margins": 1.0211350917816162, + "rewards/rejected": -3.184326171875, + "step": 13870 + }, + { + "epoch": 0.91, + "learning_rate": 1.274377541109953e-07, + "logits/chosen": -0.9694555401802063, + "logits/rejected": -1.0235732793807983, + "logps/chosen": -373.6258239746094, + "logps/rejected": -528.3242797851562, + "loss": 0.5527, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.0927529335021973, + "rewards/margins": 0.7677472233772278, + "rewards/rejected": -2.8605000972747803, + "step": 13880 + }, + { + "epoch": 0.91, + "learning_rate": 1.2564417380773435e-07, + "logits/chosen": -0.822708785533905, + "logits/rejected": -0.5482523441314697, + "logps/chosen": -407.5481872558594, + "logps/rejected": -551.6847534179688, + "loss": 0.4509, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.2905116081237793, + "rewards/margins": 1.2165449857711792, + "rewards/rejected": -3.507056713104248, + "step": 13890 + }, + { + "epoch": 0.91, + "learning_rate": 1.2386297930755436e-07, + "logits/chosen": -1.2471745014190674, + "logits/rejected": -1.1345198154449463, + "logps/chosen": -508.18096923828125, + "logps/rejected": -608.7763671875, + "loss": 0.5562, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.602952003479004, + "rewards/margins": 1.0542181730270386, + "rewards/rejected": -3.657170057296753, + "step": 13900 + }, + { + "epoch": 0.91, + "eval_logits/chosen": -1.0553574562072754, + "eval_logits/rejected": -0.9256516098976135, + "eval_logps/chosen": -433.4284362792969, + "eval_logps/rejected": -521.5325927734375, + "eval_loss": 0.48544591665267944, + "eval_rewards/accuracies": 0.6725000143051147, + "eval_rewards/chosen": -2.014234781265259, + "eval_rewards/margins": 1.0849719047546387, + "eval_rewards/rejected": -3.0992064476013184, + "eval_runtime": 734.9627, + "eval_samples_per_second": 2.721, + "eval_steps_per_second": 1.361, + "step": 13900 + }, + { + "epoch": 0.91, + "learning_rate": 1.220941799020378e-07, + "logits/chosen": -0.9485194087028503, + "logits/rejected": -0.8109304308891296, + "logps/chosen": -417.08258056640625, + "logps/rejected": -526.6878662109375, + "loss": 0.4801, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9800148010253906, + "rewards/margins": 1.319581389427185, + "rewards/rejected": -3.2995963096618652, + "step": 13910 + }, + { + "epoch": 0.91, + "learning_rate": 1.2033778481810975e-07, + "logits/chosen": -1.0876718759536743, + "logits/rejected": -1.0527876615524292, + "logps/chosen": -403.35272216796875, + "logps/rejected": -484.29534912109375, + "loss": 0.5012, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8335920572280884, + "rewards/margins": 1.182887315750122, + "rewards/rejected": -3.0164794921875, + "step": 13920 + }, + { + "epoch": 0.91, + "learning_rate": 1.1859380321798591e-07, + "logits/chosen": -1.12349271774292, + "logits/rejected": -1.3553979396820068, + "logps/chosen": -406.2502746582031, + "logps/rejected": -502.95263671875, + "loss": 0.526, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0368897914886475, + "rewards/margins": 0.8195746541023254, + "rewards/rejected": -2.8564646244049072, + "step": 13930 + }, + { + "epoch": 0.91, + "learning_rate": 1.1686224419912989e-07, + "logits/chosen": -1.007314920425415, + "logits/rejected": -0.8068926930427551, + "logps/chosen": -491.4189453125, + "logps/rejected": -588.8319091796875, + "loss": 0.5216, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.385463237762451, + "rewards/margins": 1.267038106918335, + "rewards/rejected": -3.6525015830993652, + "step": 13940 + }, + { + "epoch": 0.91, + "learning_rate": 1.1514311679420104e-07, + "logits/chosen": -0.6763644218444824, + "logits/rejected": -0.7290538549423218, + "logps/chosen": -371.91876220703125, + "logps/rejected": -528.335693359375, + "loss": 0.5789, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1677021980285645, + "rewards/margins": 1.0059720277786255, + "rewards/rejected": -3.1736743450164795, + "step": 13950 + }, + { + "epoch": 0.91, + "learning_rate": 1.1343642997101029e-07, + "logits/chosen": -1.1263518333435059, + "logits/rejected": -0.943850040435791, + "logps/chosen": -407.45391845703125, + "logps/rejected": -496.84991455078125, + "loss": 0.497, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0510058403015137, + "rewards/margins": 1.0261781215667725, + "rewards/rejected": -3.077183723449707, + "step": 13960 + }, + { + "epoch": 0.91, + "learning_rate": 1.1174219263247188e-07, + "logits/chosen": -0.659032940864563, + "logits/rejected": -0.6590057015419006, + "logps/chosen": -381.11053466796875, + "logps/rejected": -466.47900390625, + "loss": 0.4919, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.8229297399520874, + "rewards/margins": 1.028996229171753, + "rewards/rejected": -2.85192608833313, + "step": 13970 + }, + { + "epoch": 0.91, + "learning_rate": 1.1006041361655839e-07, + "logits/chosen": -1.2615095376968384, + "logits/rejected": -0.8521437644958496, + "logps/chosen": -404.3982238769531, + "logps/rejected": -455.39410400390625, + "loss": 0.526, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.9596093893051147, + "rewards/margins": 0.9093338847160339, + "rewards/rejected": -2.868943452835083, + "step": 13980 + }, + { + "epoch": 0.92, + "learning_rate": 1.0839110169625189e-07, + "logits/chosen": -0.8986303210258484, + "logits/rejected": -0.9911459684371948, + "logps/chosen": -447.0167541503906, + "logps/rejected": -582.6688842773438, + "loss": 0.4752, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.4017446041107178, + "rewards/margins": 1.4300748109817505, + "rewards/rejected": -3.831820011138916, + "step": 13990 + }, + { + "epoch": 0.92, + "learning_rate": 1.06734265579502e-07, + "logits/chosen": -1.0780047178268433, + "logits/rejected": -0.7636250257492065, + "logps/chosen": -478.39306640625, + "logps/rejected": -528.6465454101562, + "loss": 0.4542, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1687636375427246, + "rewards/margins": 1.1675869226455688, + "rewards/rejected": -3.336350202560425, + "step": 14000 + }, + { + "epoch": 0.92, + "eval_logits/chosen": -1.0565329790115356, + "eval_logits/rejected": -0.9270216226577759, + "eval_logps/chosen": -434.0428466796875, + "eval_logps/rejected": -521.0421142578125, + "eval_loss": 0.4857202172279358, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -2.020378828048706, + "eval_rewards/margins": 1.073923110961914, + "eval_rewards/rejected": -3.094302177429199, + "eval_runtime": 733.211, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 14000 + }, + { + "epoch": 0.92, + "learning_rate": 1.050899139091771e-07, + "logits/chosen": -1.2847198247909546, + "logits/rejected": -0.883298397064209, + "logps/chosen": -477.986083984375, + "logps/rejected": -561.3118896484375, + "loss": 0.3692, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.9935863018035889, + "rewards/margins": 1.298438549041748, + "rewards/rejected": -3.292024612426758, + "step": 14010 + }, + { + "epoch": 0.92, + "learning_rate": 1.0345805526302072e-07, + "logits/chosen": -1.177329659461975, + "logits/rejected": -1.022749662399292, + "logps/chosen": -404.83477783203125, + "logps/rejected": -488.59747314453125, + "loss": 0.4919, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.036724328994751, + "rewards/margins": 0.9100037813186646, + "rewards/rejected": -2.946727991104126, + "step": 14020 + }, + { + "epoch": 0.92, + "learning_rate": 1.0183869815360764e-07, + "logits/chosen": -1.1223341226577759, + "logits/rejected": -1.1892069578170776, + "logps/chosen": -397.103271484375, + "logps/rejected": -512.4364013671875, + "loss": 0.5226, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.0518977642059326, + "rewards/margins": 0.8269307017326355, + "rewards/rejected": -2.878828525543213, + "step": 14030 + }, + { + "epoch": 0.92, + "learning_rate": 1.0023185102829763e-07, + "logits/chosen": -0.8264619708061218, + "logits/rejected": -0.9047967791557312, + "logps/chosen": -467.5877380371094, + "logps/rejected": -589.7196044921875, + "loss": 0.4809, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.367655038833618, + "rewards/margins": 1.179471731185913, + "rewards/rejected": -3.5471267700195312, + "step": 14040 + }, + { + "epoch": 0.92, + "learning_rate": 9.863752226919182e-08, + "logits/chosen": -0.9276908040046692, + "logits/rejected": -0.7621574401855469, + "logps/chosen": -425.6780700683594, + "logps/rejected": -497.79925537109375, + "loss": 0.4506, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8794587850570679, + "rewards/margins": 1.376997709274292, + "rewards/rejected": -3.2564563751220703, + "step": 14050 + }, + { + "epoch": 0.92, + "learning_rate": 9.705572019309107e-08, + "logits/chosen": -1.0182292461395264, + "logits/rejected": -0.8643664121627808, + "logps/chosen": -485.4905700683594, + "logps/rejected": -562.658935546875, + "loss": 0.5055, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.1746318340301514, + "rewards/margins": 1.0865226984024048, + "rewards/rejected": -3.2611546516418457, + "step": 14060 + }, + { + "epoch": 0.92, + "learning_rate": 9.548645305144849e-08, + "logits/chosen": -1.2598340511322021, + "logits/rejected": -1.004939079284668, + "logps/chosen": -353.81964111328125, + "logps/rejected": -457.3272399902344, + "loss": 0.5245, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.8032443523406982, + "rewards/margins": 0.9953397512435913, + "rewards/rejected": -2.798584222793579, + "step": 14070 + }, + { + "epoch": 0.92, + "learning_rate": 9.392972903033149e-08, + "logits/chosen": -0.8470398187637329, + "logits/rejected": -1.0979256629943848, + "logps/chosen": -397.806884765625, + "logps/rejected": -458.0087890625, + "loss": 0.5621, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -1.7467561960220337, + "rewards/margins": 0.677453339099884, + "rewards/rejected": -2.4242093563079834, + "step": 14080 + }, + { + "epoch": 0.92, + "learning_rate": 9.238555625037449e-08, + "logits/chosen": -0.8955826759338379, + "logits/rejected": -0.8132731318473816, + "logps/chosen": -405.64996337890625, + "logps/rejected": -455.35821533203125, + "loss": 0.535, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.1494174003601074, + "rewards/margins": 0.7939003109931946, + "rewards/rejected": -2.9433178901672363, + "step": 14090 + }, + { + "epoch": 0.92, + "learning_rate": 9.085394276673903e-08, + "logits/chosen": -1.0234901905059814, + "logits/rejected": -1.0090752840042114, + "logps/chosen": -471.3775329589844, + "logps/rejected": -562.6046752929688, + "loss": 0.4657, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.032114267349243, + "rewards/margins": 1.0567667484283447, + "rewards/rejected": -3.088881015777588, + "step": 14100 + }, + { + "epoch": 0.92, + "eval_logits/chosen": -1.0566540956497192, + "eval_logits/rejected": -0.92729651927948, + "eval_logps/chosen": -432.3822021484375, + "eval_logps/rejected": -519.443115234375, + "eval_loss": 0.48553240299224854, + "eval_rewards/accuracies": 0.6694999933242798, + "eval_rewards/chosen": -2.003772020339966, + "eval_rewards/margins": 1.074540376663208, + "eval_rewards/rejected": -3.078312397003174, + "eval_runtime": 733.883, + "eval_samples_per_second": 2.725, + "eval_steps_per_second": 1.363, + "step": 14100 + }, + { + "epoch": 0.92, + "learning_rate": 8.933489656907157e-08, + "logits/chosen": -0.9661086201667786, + "logits/rejected": -1.0341233015060425, + "logps/chosen": -431.1689453125, + "logps/rejected": -524.0797119140625, + "loss": 0.6029, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.1383349895477295, + "rewards/margins": 0.6797035932540894, + "rewards/rejected": -2.8180384635925293, + "step": 14110 + }, + { + "epoch": 0.92, + "learning_rate": 8.782842558146127e-08, + "logits/chosen": -0.9491564631462097, + "logits/rejected": -0.8661885261535645, + "logps/chosen": -321.4067687988281, + "logps/rejected": -459.0927734375, + "loss": 0.3766, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.5952284336090088, + "rewards/margins": 1.3939249515533447, + "rewards/rejected": -2.9891533851623535, + "step": 14120 + }, + { + "epoch": 0.92, + "learning_rate": 8.633453766239836e-08, + "logits/chosen": -1.1976830959320068, + "logits/rejected": -1.1062679290771484, + "logps/chosen": -402.8575134277344, + "logps/rejected": -448.3551330566406, + "loss": 0.5217, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.6773631572723389, + "rewards/margins": 0.7735291123390198, + "rewards/rejected": -2.450892448425293, + "step": 14130 + }, + { + "epoch": 0.93, + "learning_rate": 8.485324060473448e-08, + "logits/chosen": -1.0934231281280518, + "logits/rejected": -0.9173671007156372, + "logps/chosen": -457.7579650878906, + "logps/rejected": -525.3034057617188, + "loss": 0.5423, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.1747584342956543, + "rewards/margins": 0.7993627190589905, + "rewards/rejected": -2.974121332168579, + "step": 14140 + }, + { + "epoch": 0.93, + "learning_rate": 8.338454213564052e-08, + "logits/chosen": -1.0765687227249146, + "logits/rejected": -0.832840085029602, + "logps/chosen": -443.53997802734375, + "logps/rejected": -558.3150634765625, + "loss": 0.4001, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1264307498931885, + "rewards/margins": 1.3196923732757568, + "rewards/rejected": -3.4461231231689453, + "step": 14150 + }, + { + "epoch": 0.93, + "learning_rate": 8.192844991656679e-08, + "logits/chosen": -0.9876272082328796, + "logits/rejected": -0.7183239459991455, + "logps/chosen": -453.659423828125, + "logps/rejected": -496.2523498535156, + "loss": 0.5436, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.1494274139404297, + "rewards/margins": 0.7900088429450989, + "rewards/rejected": -2.939436197280884, + "step": 14160 + }, + { + "epoch": 0.93, + "learning_rate": 8.048497154320434e-08, + "logits/chosen": -1.0411174297332764, + "logits/rejected": -1.1419099569320679, + "logps/chosen": -332.55426025390625, + "logps/rejected": -408.7363586425781, + "loss": 0.5648, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0300471782684326, + "rewards/margins": 0.6708464622497559, + "rewards/rejected": -2.7008938789367676, + "step": 14170 + }, + { + "epoch": 0.93, + "learning_rate": 7.905411454544265e-08, + "logits/chosen": -1.0737169981002808, + "logits/rejected": -1.0451024770736694, + "logps/chosen": -464.43511962890625, + "logps/rejected": -555.2205200195312, + "loss": 0.5399, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.279308795928955, + "rewards/margins": 0.8604310154914856, + "rewards/rejected": -3.139739513397217, + "step": 14180 + }, + { + "epoch": 0.93, + "learning_rate": 7.763588638733332e-08, + "logits/chosen": -1.0381790399551392, + "logits/rejected": -1.0612866878509521, + "logps/chosen": -481.591064453125, + "logps/rejected": -569.1793823242188, + "loss": 0.4937, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.191709041595459, + "rewards/margins": 1.0741246938705444, + "rewards/rejected": -3.265834093093872, + "step": 14190 + }, + { + "epoch": 0.93, + "learning_rate": 7.623029446704899e-08, + "logits/chosen": -1.2095810174942017, + "logits/rejected": -1.2709791660308838, + "logps/chosen": -510.81903076171875, + "logps/rejected": -631.96240234375, + "loss": 0.3963, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.993775725364685, + "rewards/margins": 1.5803800821304321, + "rewards/rejected": -3.574155330657959, + "step": 14200 + }, + { + "epoch": 0.93, + "eval_logits/chosen": -1.0539846420288086, + "eval_logits/rejected": -0.9247413277626038, + "eval_logps/chosen": -430.5838928222656, + "eval_logps/rejected": -518.67236328125, + "eval_loss": 0.4853227138519287, + "eval_rewards/accuracies": 0.6710000038146973, + "eval_rewards/chosen": -1.9857898950576782, + "eval_rewards/margins": 1.0848145484924316, + "eval_rewards/rejected": -3.0706045627593994, + "eval_runtime": 733.9148, + "eval_samples_per_second": 2.725, + "eval_steps_per_second": 1.363, + "step": 14200 + }, + { + "epoch": 0.93, + "learning_rate": 7.483734611684557e-08, + "logits/chosen": -0.8158127069473267, + "logits/rejected": -0.6502638459205627, + "logps/chosen": -426.8959045410156, + "logps/rejected": -492.63525390625, + "loss": 0.454, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.700100302696228, + "rewards/margins": 1.1990031003952026, + "rewards/rejected": -2.8991036415100098, + "step": 14210 + }, + { + "epoch": 0.93, + "learning_rate": 7.345704860302366e-08, + "logits/chosen": -1.4092390537261963, + "logits/rejected": -1.051461935043335, + "logps/chosen": -456.6201171875, + "logps/rejected": -592.5618896484375, + "loss": 0.3809, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.083674430847168, + "rewards/margins": 1.3662012815475464, + "rewards/rejected": -3.449875593185425, + "step": 14220 + }, + { + "epoch": 0.93, + "learning_rate": 7.208940912589224e-08, + "logits/chosen": -1.0438339710235596, + "logits/rejected": -0.8418275117874146, + "logps/chosen": -439.68048095703125, + "logps/rejected": -531.1444091796875, + "loss": 0.4848, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.300941228866577, + "rewards/margins": 1.2884247303009033, + "rewards/rejected": -3.5893661975860596, + "step": 14230 + }, + { + "epoch": 0.93, + "learning_rate": 7.073443481972753e-08, + "logits/chosen": -0.9687435030937195, + "logits/rejected": -0.8915712237358093, + "logps/chosen": -404.16595458984375, + "logps/rejected": -524.464599609375, + "loss": 0.5968, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.2001020908355713, + "rewards/margins": 1.007969856262207, + "rewards/rejected": -3.2080719470977783, + "step": 14240 + }, + { + "epoch": 0.93, + "learning_rate": 6.939213275274027e-08, + "logits/chosen": -1.1182448863983154, + "logits/rejected": -1.1295720338821411, + "logps/chosen": -431.3018493652344, + "logps/rejected": -505.3377990722656, + "loss": 0.4954, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.9204801321029663, + "rewards/margins": 0.9363868832588196, + "rewards/rejected": -2.8568673133850098, + "step": 14250 + }, + { + "epoch": 0.93, + "learning_rate": 6.806250992703461e-08, + "logits/chosen": -0.9891894459724426, + "logits/rejected": -0.9061968922615051, + "logps/chosen": -407.3405456542969, + "logps/rejected": -472.3490295410156, + "loss": 0.5011, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9064586162567139, + "rewards/margins": 0.9121868014335632, + "rewards/rejected": -2.81864595413208, + "step": 14260 + }, + { + "epoch": 0.93, + "learning_rate": 6.674557327857572e-08, + "logits/chosen": -1.1973614692687988, + "logits/rejected": -1.063331961631775, + "logps/chosen": -454.7339782714844, + "logps/rejected": -570.90673828125, + "loss": 0.4754, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0046329498291016, + "rewards/margins": 1.252607822418213, + "rewards/rejected": -3.2572407722473145, + "step": 14270 + }, + { + "epoch": 0.93, + "learning_rate": 6.544132967714917e-08, + "logits/chosen": -0.838614821434021, + "logits/rejected": -0.762345016002655, + "logps/chosen": -484.97625732421875, + "logps/rejected": -575.3137817382812, + "loss": 0.6345, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3811631202697754, + "rewards/margins": 1.0404995679855347, + "rewards/rejected": -3.4216628074645996, + "step": 14280 + }, + { + "epoch": 0.93, + "learning_rate": 6.414978592632932e-08, + "logits/chosen": -0.9945710301399231, + "logits/rejected": -0.9295830726623535, + "logps/chosen": -472.0386657714844, + "logps/rejected": -528.8554077148438, + "loss": 0.4747, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.109393835067749, + "rewards/margins": 1.032721757888794, + "rewards/rejected": -3.142115592956543, + "step": 14290 + }, + { + "epoch": 0.94, + "learning_rate": 6.287094876344046e-08, + "logits/chosen": -1.283722162246704, + "logits/rejected": -1.1530678272247314, + "logps/chosen": -319.38885498046875, + "logps/rejected": -428.88323974609375, + "loss": 0.4414, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.469685673713684, + "rewards/margins": 1.0285536050796509, + "rewards/rejected": -2.498239278793335, + "step": 14300 + }, + { + "epoch": 0.94, + "eval_logits/chosen": -1.0557368993759155, + "eval_logits/rejected": -0.9262323975563049, + "eval_logps/chosen": -431.466552734375, + "eval_logps/rejected": -519.5145263671875, + "eval_loss": 0.4854941666126251, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -1.994615912437439, + "eval_rewards/margins": 1.0844100713729858, + "eval_rewards/rejected": -3.079025983810425, + "eval_runtime": 734.1054, + "eval_samples_per_second": 2.724, + "eval_steps_per_second": 1.362, + "step": 14300 + }, + { + "epoch": 0.94, + "learning_rate": 6.160482485952413e-08, + "logits/chosen": -1.2454659938812256, + "logits/rejected": -1.209099531173706, + "logps/chosen": -450.26055908203125, + "logps/rejected": -511.4938049316406, + "loss": 0.5019, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.1381077766418457, + "rewards/margins": 0.9302912950515747, + "rewards/rejected": -3.068398952484131, + "step": 14310 + }, + { + "epoch": 0.94, + "learning_rate": 6.035142081930234e-08, + "logits/chosen": -1.0615335702896118, + "logits/rejected": -0.8751511573791504, + "logps/chosen": -484.47412109375, + "logps/rejected": -493.105712890625, + "loss": 0.4779, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.2167773246765137, + "rewards/margins": 0.8564853668212891, + "rewards/rejected": -3.0732626914978027, + "step": 14320 + }, + { + "epoch": 0.94, + "learning_rate": 5.911074318114496e-08, + "logits/chosen": -0.9687420129776001, + "logits/rejected": -0.8334226608276367, + "logps/chosen": -415.8157653808594, + "logps/rejected": -576.0595703125, + "loss": 0.4533, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.122276544570923, + "rewards/margins": 1.1811892986297607, + "rewards/rejected": -3.3034660816192627, + "step": 14330 + }, + { + "epoch": 0.94, + "learning_rate": 5.788279841703381e-08, + "logits/chosen": -1.2568345069885254, + "logits/rejected": -0.9866706728935242, + "logps/chosen": -372.8052062988281, + "logps/rejected": -492.4381408691406, + "loss": 0.4512, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.888956069946289, + "rewards/margins": 1.2730000019073486, + "rewards/rejected": -3.1619560718536377, + "step": 14340 + }, + { + "epoch": 0.94, + "learning_rate": 5.66675929325311e-08, + "logits/chosen": -1.1829434633255005, + "logits/rejected": -0.9463958740234375, + "logps/chosen": -423.58984375, + "logps/rejected": -519.6142578125, + "loss": 0.5006, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0292537212371826, + "rewards/margins": 1.0737316608428955, + "rewards/rejected": -3.1029858589172363, + "step": 14350 + }, + { + "epoch": 0.94, + "learning_rate": 5.546513306674301e-08, + "logits/chosen": -0.8857857584953308, + "logits/rejected": -0.7765743136405945, + "logps/chosen": -510.887939453125, + "logps/rejected": -541.4619140625, + "loss": 0.555, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.3237123489379883, + "rewards/margins": 1.0565690994262695, + "rewards/rejected": -3.380281925201416, + "step": 14360 + }, + { + "epoch": 0.94, + "learning_rate": 5.4275425092290004e-08, + "logits/chosen": -1.5319944620132446, + "logits/rejected": -1.317714810371399, + "logps/chosen": -457.83807373046875, + "logps/rejected": -551.9368896484375, + "loss": 0.4606, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.0304198265075684, + "rewards/margins": 1.1254143714904785, + "rewards/rejected": -3.155834197998047, + "step": 14370 + }, + { + "epoch": 0.94, + "learning_rate": 5.309847521527078e-08, + "logits/chosen": -0.78449946641922, + "logits/rejected": -0.6805815100669861, + "logps/chosen": -494.6311950683594, + "logps/rejected": -557.3088989257812, + "loss": 0.4461, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.100578784942627, + "rewards/margins": 1.0560588836669922, + "rewards/rejected": -3.156637668609619, + "step": 14380 + }, + { + "epoch": 0.94, + "learning_rate": 5.1934289575233385e-08, + "logits/chosen": -0.8264106512069702, + "logits/rejected": -0.606268584728241, + "logps/chosen": -445.46697998046875, + "logps/rejected": -517.4832763671875, + "loss": 0.4771, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0593760013580322, + "rewards/margins": 1.0638360977172852, + "rewards/rejected": -3.1232123374938965, + "step": 14390 + }, + { + "epoch": 0.94, + "learning_rate": 5.078287424513994e-08, + "logits/chosen": -1.2722312211990356, + "logits/rejected": -1.099533200263977, + "logps/chosen": -480.6007385253906, + "logps/rejected": -511.64013671875, + "loss": 0.5011, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.10552978515625, + "rewards/margins": 1.0586974620819092, + "rewards/rejected": -3.164227247238159, + "step": 14400 + }, + { + "epoch": 0.94, + "eval_logits/chosen": -1.0528078079223633, + "eval_logits/rejected": -0.9236515164375305, + "eval_logps/chosen": -431.91925048828125, + "eval_logps/rejected": -520.1353759765625, + "eval_loss": 0.48540934920310974, + "eval_rewards/accuracies": 0.6725000143051147, + "eval_rewards/chosen": -1.999143362045288, + "eval_rewards/margins": 1.0860916376113892, + "eval_rewards/rejected": -3.085235118865967, + "eval_runtime": 732.8713, + "eval_samples_per_second": 2.729, + "eval_steps_per_second": 1.364, + "step": 14400 + }, + { + "epoch": 0.94, + "learning_rate": 4.964423523133671e-08, + "logits/chosen": -1.3805128335952759, + "logits/rejected": -0.9356088638305664, + "logps/chosen": -385.4630126953125, + "logps/rejected": -447.2681579589844, + "loss": 0.4266, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.677315354347229, + "rewards/margins": 0.9615432024002075, + "rewards/rejected": -2.6388583183288574, + "step": 14410 + }, + { + "epoch": 0.94, + "learning_rate": 4.8518378473522976e-08, + "logits/chosen": -1.160941481590271, + "logits/rejected": -0.9066068530082703, + "logps/chosen": -430.5773010253906, + "logps/rejected": -544.8810424804688, + "loss": 0.5317, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.8006746768951416, + "rewards/margins": 1.210309386253357, + "rewards/rejected": -3.010983943939209, + "step": 14420 + }, + { + "epoch": 0.94, + "learning_rate": 4.7405309844718584e-08, + "logits/chosen": -1.129230260848999, + "logits/rejected": -0.9649642109870911, + "logps/chosen": -415.31048583984375, + "logps/rejected": -553.1641845703125, + "loss": 0.4613, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.228114366531372, + "rewards/margins": 1.2865746021270752, + "rewards/rejected": -3.5146892070770264, + "step": 14430 + }, + { + "epoch": 0.94, + "learning_rate": 4.630503515123508e-08, + "logits/chosen": -1.2513015270233154, + "logits/rejected": -0.9588994979858398, + "logps/chosen": -391.487060546875, + "logps/rejected": -449.8251953125, + "loss": 0.4846, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.90621018409729, + "rewards/margins": 1.0820457935333252, + "rewards/rejected": -2.9882559776306152, + "step": 14440 + }, + { + "epoch": 0.95, + "learning_rate": 4.5217560132644056e-08, + "logits/chosen": -0.9577115774154663, + "logits/rejected": -0.7936094403266907, + "logps/chosen": -338.69146728515625, + "logps/rejected": -458.49383544921875, + "loss": 0.5356, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8807188272476196, + "rewards/margins": 1.0140142440795898, + "rewards/rejected": -2.89473295211792, + "step": 14450 + }, + { + "epoch": 0.95, + "learning_rate": 4.41428904617483e-08, + "logits/chosen": -1.065185308456421, + "logits/rejected": -1.139467716217041, + "logps/chosen": -379.53497314453125, + "logps/rejected": -466.9539489746094, + "loss": 0.5526, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -2.0242626667022705, + "rewards/margins": 0.8275849223136902, + "rewards/rejected": -2.8518476486206055, + "step": 14460 + }, + { + "epoch": 0.95, + "learning_rate": 4.3081031744550696e-08, + "logits/chosen": -1.1921031475067139, + "logits/rejected": -1.2130444049835205, + "logps/chosen": -435.697265625, + "logps/rejected": -533.8173828125, + "loss": 0.417, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8255189657211304, + "rewards/margins": 1.2044792175292969, + "rewards/rejected": -3.029998302459717, + "step": 14470 + }, + { + "epoch": 0.95, + "learning_rate": 4.2031989520227025e-08, + "logits/chosen": -1.0036526918411255, + "logits/rejected": -0.9116290807723999, + "logps/chosen": -423.17938232421875, + "logps/rejected": -516.8751831054688, + "loss": 0.3985, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.9486496448516846, + "rewards/margins": 1.1558328866958618, + "rewards/rejected": -3.104482650756836, + "step": 14480 + }, + { + "epoch": 0.95, + "learning_rate": 4.099576926109461e-08, + "logits/chosen": -1.3175681829452515, + "logits/rejected": -0.9569937586784363, + "logps/chosen": -435.8815002441406, + "logps/rejected": -468.4258728027344, + "loss": 0.5283, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.9609966278076172, + "rewards/margins": 1.116987943649292, + "rewards/rejected": -3.077984571456909, + "step": 14490 + }, + { + "epoch": 0.95, + "learning_rate": 3.997237637258705e-08, + "logits/chosen": -1.1571390628814697, + "logits/rejected": -0.9494622349739075, + "logps/chosen": -518.707763671875, + "logps/rejected": -591.4268798828125, + "loss": 0.4677, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.0145442485809326, + "rewards/margins": 1.1606231927871704, + "rewards/rejected": -3.1751675605773926, + "step": 14500 + }, + { + "epoch": 0.95, + "eval_logits/chosen": -1.0542787313461304, + "eval_logits/rejected": -0.924854040145874, + "eval_logps/chosen": -432.1260986328125, + "eval_logps/rejected": -520.5852661132812, + "eval_loss": 0.48530498147010803, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -2.001211643218994, + "eval_rewards/margins": 1.08852219581604, + "eval_rewards/rejected": -3.089733839035034, + "eval_runtime": 734.0536, + "eval_samples_per_second": 2.725, + "eval_steps_per_second": 1.362, + "step": 14500 + }, + { + "epoch": 0.95, + "learning_rate": 3.8961816193222035e-08, + "logits/chosen": -1.1297836303710938, + "logits/rejected": -0.8937932252883911, + "logps/chosen": -470.29498291015625, + "logps/rejected": -474.1014709472656, + "loss": 0.6551, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.344447374343872, + "rewards/margins": 0.6113969087600708, + "rewards/rejected": -2.9558444023132324, + "step": 14510 + }, + { + "epoch": 0.95, + "learning_rate": 3.79640939945769e-08, + "logits/chosen": -1.0833925008773804, + "logits/rejected": -0.9126625061035156, + "logps/chosen": -436.407958984375, + "logps/rejected": -439.82403564453125, + "loss": 0.4772, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.5018702745437622, + "rewards/margins": 0.8968018293380737, + "rewards/rejected": -2.398672103881836, + "step": 14520 + }, + { + "epoch": 0.95, + "learning_rate": 3.697921498125895e-08, + "logits/chosen": -0.9268239140510559, + "logits/rejected": -1.0825777053833008, + "logps/chosen": -411.1507873535156, + "logps/rejected": -523.050537109375, + "loss": 0.5322, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0119776725769043, + "rewards/margins": 1.060199499130249, + "rewards/rejected": -3.0721774101257324, + "step": 14530 + }, + { + "epoch": 0.95, + "learning_rate": 3.6007184290880456e-08, + "logits/chosen": -1.1957523822784424, + "logits/rejected": -1.0776623487472534, + "logps/chosen": -448.6844177246094, + "logps/rejected": -524.9656982421875, + "loss": 0.6034, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.367114305496216, + "rewards/margins": 0.9294492602348328, + "rewards/rejected": -3.2965633869171143, + "step": 14540 + }, + { + "epoch": 0.95, + "learning_rate": 3.504800699402872e-08, + "logits/chosen": -1.2815639972686768, + "logits/rejected": -1.082935094833374, + "logps/chosen": -589.32470703125, + "logps/rejected": -599.1885986328125, + "loss": 0.6407, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.4367051124572754, + "rewards/margins": 0.8323081731796265, + "rewards/rejected": -3.2690131664276123, + "step": 14550 + }, + { + "epoch": 0.95, + "learning_rate": 3.4101688094242967e-08, + "logits/chosen": -1.0604369640350342, + "logits/rejected": -0.8951923251152039, + "logps/chosen": -520.3701782226562, + "logps/rejected": -614.2811279296875, + "loss": 0.5229, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.508091926574707, + "rewards/margins": 1.1908717155456543, + "rewards/rejected": -3.6989638805389404, + "step": 14560 + }, + { + "epoch": 0.95, + "learning_rate": 3.3168232527985564e-08, + "logits/chosen": -0.7122337222099304, + "logits/rejected": -0.692426860332489, + "logps/chosen": -414.7545471191406, + "logps/rejected": -489.59112548828125, + "loss": 0.487, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.731359839439392, + "rewards/margins": 1.2958645820617676, + "rewards/rejected": -3.02722430229187, + "step": 14570 + }, + { + "epoch": 0.95, + "learning_rate": 3.224764516461892e-08, + "logits/chosen": -1.1161458492279053, + "logits/rejected": -0.8429574966430664, + "logps/chosen": -435.79296875, + "logps/rejected": -527.206298828125, + "loss": 0.4533, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.807941198348999, + "rewards/margins": 1.221619963645935, + "rewards/rejected": -3.0295608043670654, + "step": 14580 + }, + { + "epoch": 0.95, + "learning_rate": 3.133993080637665e-08, + "logits/chosen": -1.1565264463424683, + "logits/rejected": -0.911404013633728, + "logps/chosen": -393.412841796875, + "logps/rejected": -508.4385681152344, + "loss": 0.478, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9053192138671875, + "rewards/margins": 1.2743666172027588, + "rewards/rejected": -3.1796860694885254, + "step": 14590 + }, + { + "epoch": 0.96, + "learning_rate": 3.0445094188342186e-08, + "logits/chosen": -0.5498150587081909, + "logits/rejected": -0.4293433725833893, + "logps/chosen": -443.096923828125, + "logps/rejected": -475.79681396484375, + "loss": 0.4234, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.8962033987045288, + "rewards/margins": 1.1296409368515015, + "rewards/rejected": -3.025844097137451, + "step": 14600 + }, + { + "epoch": 0.96, + "eval_logits/chosen": -1.05791175365448, + "eval_logits/rejected": -0.9282616376876831, + "eval_logps/chosen": -432.10369873046875, + "eval_logps/rejected": -520.2671508789062, + "eval_loss": 0.48540204763412476, + "eval_rewards/accuracies": 0.6710000038146973, + "eval_rewards/chosen": -2.0009877681732178, + "eval_rewards/margins": 1.0855647325515747, + "eval_rewards/rejected": -3.086552143096924, + "eval_runtime": 734.9099, + "eval_samples_per_second": 2.721, + "eval_steps_per_second": 1.361, + "step": 14600 + }, + { + "epoch": 0.96, + "learning_rate": 2.9563139978421028e-08, + "logits/chosen": -0.9228948354721069, + "logits/rejected": -0.9851576089859009, + "logps/chosen": -426.33856201171875, + "logps/rejected": -487.5555725097656, + "loss": 0.6203, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.004666805267334, + "rewards/margins": 0.6900150179862976, + "rewards/rejected": -2.6946821212768555, + "step": 14610 + }, + { + "epoch": 0.96, + "learning_rate": 2.869407277731939e-08, + "logits/chosen": -0.7606264352798462, + "logits/rejected": -0.7889705896377563, + "logps/chosen": -361.2055969238281, + "logps/rejected": -435.94757080078125, + "loss": 0.4694, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.759484052658081, + "rewards/margins": 0.9791158437728882, + "rewards/rejected": -2.738600254058838, + "step": 14620 + }, + { + "epoch": 0.96, + "learning_rate": 2.783789711851642e-08, + "logits/chosen": -1.157167911529541, + "logits/rejected": -0.877331554889679, + "logps/chosen": -355.41644287109375, + "logps/rejected": -471.55120849609375, + "loss": 0.4778, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9356542825698853, + "rewards/margins": 1.385674238204956, + "rewards/rejected": -3.321328639984131, + "step": 14630 + }, + { + "epoch": 0.96, + "learning_rate": 2.6994617468244778e-08, + "logits/chosen": -1.0710012912750244, + "logits/rejected": -0.9255205988883972, + "logps/chosen": -413.6026916503906, + "logps/rejected": -466.4881896972656, + "loss": 0.4256, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0058205127716064, + "rewards/margins": 1.1680415868759155, + "rewards/rejected": -3.1738619804382324, + "step": 14640 + }, + { + "epoch": 0.96, + "learning_rate": 2.6164238225463155e-08, + "logits/chosen": -0.9301780462265015, + "logits/rejected": -0.6651344299316406, + "logps/chosen": -469.53314208984375, + "logps/rejected": -512.8493041992188, + "loss": 0.4997, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8710609674453735, + "rewards/margins": 1.2182409763336182, + "rewards/rejected": -3.089301824569702, + "step": 14650 + }, + { + "epoch": 0.96, + "learning_rate": 2.534676372183742e-08, + "logits/chosen": -0.8242905735969543, + "logits/rejected": -0.8369461894035339, + "logps/chosen": -473.88983154296875, + "logps/rejected": -563.2259521484375, + "loss": 0.5052, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.8993546962738037, + "rewards/margins": 1.4166282415390015, + "rewards/rejected": -3.315983295440674, + "step": 14660 + }, + { + "epoch": 0.96, + "learning_rate": 2.4542198221714218e-08, + "logits/chosen": -0.7648957967758179, + "logits/rejected": -0.6464065313339233, + "logps/chosen": -311.3927307128906, + "logps/rejected": -418.7562561035156, + "loss": 0.5343, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.7328574657440186, + "rewards/margins": 1.0713664293289185, + "rewards/rejected": -2.8042235374450684, + "step": 14670 + }, + { + "epoch": 0.96, + "learning_rate": 2.3750545922101854e-08, + "logits/chosen": -1.4671556949615479, + "logits/rejected": -0.8755186200141907, + "logps/chosen": -518.0368041992188, + "logps/rejected": -562.1136474609375, + "loss": 0.4419, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.086455821990967, + "rewards/margins": 1.1247096061706543, + "rewards/rejected": -3.2111656665802, + "step": 14680 + }, + { + "epoch": 0.96, + "learning_rate": 2.2971810952646112e-08, + "logits/chosen": -1.1903879642486572, + "logits/rejected": -0.998426616191864, + "logps/chosen": -480.20343017578125, + "logps/rejected": -508.8087463378906, + "loss": 0.566, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.2184062004089355, + "rewards/margins": 0.7631117701530457, + "rewards/rejected": -2.981517791748047, + "step": 14690 + }, + { + "epoch": 0.96, + "learning_rate": 2.2205997375610576e-08, + "logits/chosen": -0.8135994076728821, + "logits/rejected": -0.7705693244934082, + "logps/chosen": -336.8621826171875, + "logps/rejected": -462.5765075683594, + "loss": 0.4681, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.513562798500061, + "rewards/margins": 1.2055425643920898, + "rewards/rejected": -2.7191052436828613, + "step": 14700 + }, + { + "epoch": 0.96, + "eval_logits/chosen": -1.055994987487793, + "eval_logits/rejected": -0.9266590476036072, + "eval_logps/chosen": -431.9801025390625, + "eval_logps/rejected": -520.0926513671875, + "eval_loss": 0.4855150580406189, + "eval_rewards/accuracies": 0.6700000166893005, + "eval_rewards/chosen": -1.9997512102127075, + "eval_rewards/margins": 1.0850558280944824, + "eval_rewards/rejected": -3.0848069190979004, + "eval_runtime": 733.2428, + "eval_samples_per_second": 2.728, + "eval_steps_per_second": 1.364, + "step": 14700 + }, + { + "epoch": 0.96, + "learning_rate": 2.1453109185853304e-08, + "logits/chosen": -1.1346324682235718, + "logits/rejected": -1.1408488750457764, + "logps/chosen": -374.3671569824219, + "logps/rejected": -476.6963806152344, + "loss": 0.4865, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7228870391845703, + "rewards/margins": 1.005819320678711, + "rewards/rejected": -2.7287065982818604, + "step": 14710 + }, + { + "epoch": 0.96, + "learning_rate": 2.0713150310808784e-08, + "logits/chosen": -1.140235185623169, + "logits/rejected": -1.0906654596328735, + "logps/chosen": -420.65869140625, + "logps/rejected": -480.44708251953125, + "loss": 0.5937, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.9517085552215576, + "rewards/margins": 0.5402504205703735, + "rewards/rejected": -2.4919590950012207, + "step": 14720 + }, + { + "epoch": 0.96, + "learning_rate": 1.9986124610464064e-08, + "logits/chosen": -0.7904390096664429, + "logits/rejected": -0.6426718235015869, + "logps/chosen": -528.2882690429688, + "logps/rejected": -612.8759155273438, + "loss": 0.3436, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -2.410581111907959, + "rewards/margins": 1.5439999103546143, + "rewards/rejected": -3.954580783843994, + "step": 14730 + }, + { + "epoch": 0.96, + "learning_rate": 1.927203587734211e-08, + "logits/chosen": -0.7250292897224426, + "logits/rejected": -0.6967679262161255, + "logps/chosen": -438.3182067871094, + "logps/rejected": -526.4837646484375, + "loss": 0.5014, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.8532650470733643, + "rewards/margins": 1.366919994354248, + "rewards/rejected": -3.2201850414276123, + "step": 14740 + }, + { + "epoch": 0.97, + "learning_rate": 1.8570887836479034e-08, + "logits/chosen": -0.9606091380119324, + "logits/rejected": -0.7177838087081909, + "logps/chosen": -425.73175048828125, + "logps/rejected": -569.6988525390625, + "loss": 0.4986, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -2.3210859298706055, + "rewards/margins": 0.9115177989006042, + "rewards/rejected": -3.2326037883758545, + "step": 14750 + }, + { + "epoch": 0.97, + "learning_rate": 1.7882684145406616e-08, + "logits/chosen": -1.1041526794433594, + "logits/rejected": -1.0806796550750732, + "logps/chosen": -485.91790771484375, + "logps/rejected": -578.8372802734375, + "loss": 0.5158, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.8949730396270752, + "rewards/margins": 0.9480817914009094, + "rewards/rejected": -2.84305477142334, + "step": 14760 + }, + { + "epoch": 0.97, + "learning_rate": 1.7207428394132865e-08, + "logits/chosen": -1.3055307865142822, + "logits/rejected": -0.9931842088699341, + "logps/chosen": -477.50469970703125, + "logps/rejected": -567.6167602539062, + "loss": 0.4226, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.0772435665130615, + "rewards/margins": 1.447993516921997, + "rewards/rejected": -3.5252373218536377, + "step": 14770 + }, + { + "epoch": 0.97, + "learning_rate": 1.654512410512177e-08, + "logits/chosen": -1.0979361534118652, + "logits/rejected": -0.8360413312911987, + "logps/chosen": -447.86029052734375, + "logps/rejected": -473.5550842285156, + "loss": 0.5438, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.9428339004516602, + "rewards/margins": 0.8808323740959167, + "rewards/rejected": -2.8236660957336426, + "step": 14780 + }, + { + "epoch": 0.97, + "learning_rate": 1.5895774733277468e-08, + "logits/chosen": -0.984605610370636, + "logits/rejected": -0.8599230051040649, + "logps/chosen": -512.8638916015625, + "logps/rejected": -566.1119384765625, + "loss": 0.5088, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.3304696083068848, + "rewards/margins": 1.0348457098007202, + "rewards/rejected": -3.3653149604797363, + "step": 14790 + }, + { + "epoch": 0.97, + "learning_rate": 1.5259383665924e-08, + "logits/chosen": -1.5153400897979736, + "logits/rejected": -1.1625539064407349, + "logps/chosen": -537.54150390625, + "logps/rejected": -583.3179931640625, + "loss": 0.4417, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.954496145248413, + "rewards/margins": 1.3978536128997803, + "rewards/rejected": -3.3523495197296143, + "step": 14800 + }, + { + "epoch": 0.97, + "eval_logits/chosen": -1.0548804998397827, + "eval_logits/rejected": -0.9253528118133545, + "eval_logps/chosen": -432.188232421875, + "eval_logps/rejected": -520.3868408203125, + "eval_loss": 0.48528552055358887, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -2.0018327236175537, + "eval_rewards/margins": 1.0859160423278809, + "eval_rewards/rejected": -3.0877487659454346, + "eval_runtime": 735.1276, + "eval_samples_per_second": 2.721, + "eval_steps_per_second": 1.36, + "step": 14800 + }, + { + "epoch": 0.97, + "learning_rate": 1.4635954222789461e-08, + "logits/chosen": -1.218711256980896, + "logits/rejected": -1.1651209592819214, + "logps/chosen": -416.23004150390625, + "logps/rejected": -518.4617919921875, + "loss": 0.5465, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -1.9912240505218506, + "rewards/margins": 0.971238911151886, + "rewards/rejected": -2.962462902069092, + "step": 14810 + }, + { + "epoch": 0.97, + "learning_rate": 1.402548965598688e-08, + "logits/chosen": -1.1414233446121216, + "logits/rejected": -0.9460879564285278, + "logps/chosen": -413.87188720703125, + "logps/rejected": -520.7072143554688, + "loss": 0.4791, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -2.0924458503723145, + "rewards/margins": 1.1005178689956665, + "rewards/rejected": -3.1929638385772705, + "step": 14820 + }, + { + "epoch": 0.97, + "learning_rate": 1.3427993149998375e-08, + "logits/chosen": -1.2117063999176025, + "logits/rejected": -1.0982780456542969, + "logps/chosen": -437.361083984375, + "logps/rejected": -491.078125, + "loss": 0.4716, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.9532392024993896, + "rewards/margins": 1.0908714532852173, + "rewards/rejected": -3.0441105365753174, + "step": 14830 + }, + { + "epoch": 0.97, + "learning_rate": 1.2843467821658518e-08, + "logits/chosen": -1.2357401847839355, + "logits/rejected": -1.0914043188095093, + "logps/chosen": -398.68353271484375, + "logps/rejected": -521.7628784179688, + "loss": 0.4396, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7246061563491821, + "rewards/margins": 1.2344787120819092, + "rewards/rejected": -2.959084987640381, + "step": 14840 + }, + { + "epoch": 0.97, + "learning_rate": 1.2271916720137666e-08, + "logits/chosen": -1.4540129899978638, + "logits/rejected": -1.137785792350769, + "logps/chosen": -503.82501220703125, + "logps/rejected": -536.3612060546875, + "loss": 0.5983, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.1805543899536133, + "rewards/margins": 0.786711573600769, + "rewards/rejected": -2.967266082763672, + "step": 14850 + }, + { + "epoch": 0.97, + "learning_rate": 1.171334282692671e-08, + "logits/chosen": -1.2715617418289185, + "logits/rejected": -1.0421197414398193, + "logps/chosen": -478.9646911621094, + "logps/rejected": -586.7308959960938, + "loss": 0.396, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9636627435684204, + "rewards/margins": 1.3839813470840454, + "rewards/rejected": -3.3476436138153076, + "step": 14860 + }, + { + "epoch": 0.97, + "learning_rate": 1.116774905582041e-08, + "logits/chosen": -1.1228783130645752, + "logits/rejected": -0.9871054887771606, + "logps/chosen": -374.7401428222656, + "logps/rejected": -438.5972595214844, + "loss": 0.5955, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.8992159366607666, + "rewards/margins": 0.7656039595603943, + "rewards/rejected": -2.6648201942443848, + "step": 14870 + }, + { + "epoch": 0.97, + "learning_rate": 1.0635138252902966e-08, + "logits/chosen": -1.2849915027618408, + "logits/rejected": -1.150407075881958, + "logps/chosen": -429.98321533203125, + "logps/rejected": -527.2002563476562, + "loss": 0.4301, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.995692491531372, + "rewards/margins": 1.1934545040130615, + "rewards/rejected": -3.1891469955444336, + "step": 14880 + }, + { + "epoch": 0.97, + "learning_rate": 1.0115513196533589e-08, + "logits/chosen": -1.2216876745224, + "logits/rejected": -1.0180718898773193, + "logps/chosen": -453.65777587890625, + "logps/rejected": -507.4833068847656, + "loss": 0.5853, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9104763269424438, + "rewards/margins": 0.7113370895385742, + "rewards/rejected": -2.6218130588531494, + "step": 14890 + }, + { + "epoch": 0.97, + "learning_rate": 9.608876597330952e-09, + "logits/chosen": -1.1359317302703857, + "logits/rejected": -0.8752700090408325, + "logps/chosen": -544.45068359375, + "logps/rejected": -649.2925415039062, + "loss": 0.516, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.5782792568206787, + "rewards/margins": 1.1652708053588867, + "rewards/rejected": -3.7435498237609863, + "step": 14900 + }, + { + "epoch": 0.97, + "eval_logits/chosen": -1.0542715787887573, + "eval_logits/rejected": -0.9249005317687988, + "eval_logps/chosen": -432.13201904296875, + "eval_logps/rejected": -520.3480834960938, + "eval_loss": 0.48537370562553406, + "eval_rewards/accuracies": 0.6700000166893005, + "eval_rewards/chosen": -2.0012712478637695, + "eval_rewards/margins": 1.086090087890625, + "eval_rewards/rejected": -3.0873613357543945, + "eval_runtime": 732.6647, + "eval_samples_per_second": 2.73, + "eval_steps_per_second": 1.365, + "step": 14900 + }, + { + "epoch": 0.98, + "learning_rate": 9.115231098159594e-09, + "logits/chosen": -1.2317795753479004, + "logits/rejected": -1.197540283203125, + "logps/chosen": -445.7748107910156, + "logps/rejected": -509.39288330078125, + "loss": 0.5767, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9246104955673218, + "rewards/margins": 0.8392006754875183, + "rewards/rejected": -2.7638113498687744, + "step": 14910 + }, + { + "epoch": 0.98, + "learning_rate": 8.634579274116317e-09, + "logits/chosen": -1.0000141859054565, + "logits/rejected": -1.0341566801071167, + "logps/chosen": -363.2093200683594, + "logps/rejected": -465.8623046875, + "loss": 0.6092, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.7315008640289307, + "rewards/margins": 0.8204340934753418, + "rewards/rejected": -2.5519349575042725, + "step": 14920 + }, + { + "epoch": 0.98, + "learning_rate": 8.166923632516865e-09, + "logits/chosen": -1.2367180585861206, + "logits/rejected": -1.001817226409912, + "logps/chosen": -417.31402587890625, + "logps/rejected": -633.681884765625, + "loss": 0.3654, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.8958278894424438, + "rewards/margins": 1.787407636642456, + "rewards/rejected": -3.6832358837127686, + "step": 14930 + }, + { + "epoch": 0.98, + "learning_rate": 7.712266612881492e-09, + "logits/chosen": -0.8679588437080383, + "logits/rejected": -0.7478830218315125, + "logps/chosen": -344.3149719238281, + "logps/rejected": -457.35089111328125, + "loss": 0.4323, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.5907829999923706, + "rewards/margins": 1.1791170835494995, + "rewards/rejected": -2.769899845123291, + "step": 14940 + }, + { + "epoch": 0.98, + "learning_rate": 7.270610586924687e-09, + "logits/chosen": -1.2399359941482544, + "logits/rejected": -1.1076147556304932, + "logps/chosen": -456.895751953125, + "logps/rejected": -520.5778198242188, + "loss": 0.4983, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.919304609298706, + "rewards/margins": 1.0776231288909912, + "rewards/rejected": -2.996927499771118, + "step": 14950 + }, + { + "epoch": 0.98, + "learning_rate": 6.841957858539916e-09, + "logits/chosen": -1.0050383806228638, + "logits/rejected": -0.9870063662528992, + "logps/chosen": -399.08941650390625, + "logps/rejected": -478.54241943359375, + "loss": 0.5808, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.324338912963867, + "rewards/margins": 0.6860220432281494, + "rewards/rejected": -3.0103607177734375, + "step": 14960 + }, + { + "epoch": 0.98, + "learning_rate": 6.426310663790181e-09, + "logits/chosen": -0.9850949048995972, + "logits/rejected": -0.6555286645889282, + "logps/chosen": -460.75439453125, + "logps/rejected": -519.1944580078125, + "loss": 0.5963, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.237785816192627, + "rewards/margins": 0.8990850448608398, + "rewards/rejected": -3.136870861053467, + "step": 14970 + }, + { + "epoch": 0.98, + "learning_rate": 6.023671170894696e-09, + "logits/chosen": -1.3116604089736938, + "logits/rejected": -0.957747757434845, + "logps/chosen": -485.274658203125, + "logps/rejected": -561.7952880859375, + "loss": 0.3114, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.929460883140564, + "rewards/margins": 1.6330032348632812, + "rewards/rejected": -3.5624642372131348, + "step": 14980 + }, + { + "epoch": 0.98, + "learning_rate": 5.634041480218344e-09, + "logits/chosen": -1.0634709596633911, + "logits/rejected": -1.2313354015350342, + "logps/chosen": -450.6940002441406, + "logps/rejected": -563.3707885742188, + "loss": 0.4649, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9762039184570312, + "rewards/margins": 1.1711724996566772, + "rewards/rejected": -3.147376298904419, + "step": 14990 + }, + { + "epoch": 0.98, + "learning_rate": 5.257423624260849e-09, + "logits/chosen": -1.2537717819213867, + "logits/rejected": -0.9475277066230774, + "logps/chosen": -469.4458923339844, + "logps/rejected": -545.3299560546875, + "loss": 0.5369, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.0612242221832275, + "rewards/margins": 1.1912734508514404, + "rewards/rejected": -3.252497911453247, + "step": 15000 + }, + { + "epoch": 0.98, + "eval_logits/chosen": -1.0536617040634155, + "eval_logits/rejected": -0.924413800239563, + "eval_logps/chosen": -432.14788818359375, + "eval_logps/rejected": -520.3270874023438, + "eval_loss": 0.4853664040565491, + "eval_rewards/accuracies": 0.6704999804496765, + "eval_rewards/chosen": -2.0014290809631348, + "eval_rewards/margins": 1.085722804069519, + "eval_rewards/rejected": -3.0871517658233643, + "eval_runtime": 731.2965, + "eval_samples_per_second": 2.735, + "eval_steps_per_second": 1.367, + "step": 15000 + }, + { + "epoch": 0.98, + "learning_rate": 4.893819567644564e-09, + "logits/chosen": -0.9579187631607056, + "logits/rejected": -0.9031054377555847, + "logps/chosen": -366.45526123046875, + "logps/rejected": -449.436767578125, + "loss": 0.5813, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.7845852375030518, + "rewards/margins": 0.7280542850494385, + "rewards/rejected": -2.5126395225524902, + "step": 15010 + }, + { + "epoch": 0.98, + "learning_rate": 4.543231207107257e-09, + "logits/chosen": -0.8927432894706726, + "logits/rejected": -0.96844881772995, + "logps/chosen": -475.09466552734375, + "logps/rejected": -545.72900390625, + "loss": 0.5459, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -2.1727311611175537, + "rewards/margins": 0.9424299001693726, + "rewards/rejected": -3.115161180496216, + "step": 15020 + }, + { + "epoch": 0.98, + "learning_rate": 4.205660371488785e-09, + "logits/chosen": -1.2815234661102295, + "logits/rejected": -1.1932264566421509, + "logps/chosen": -508.7298278808594, + "logps/rejected": -565.9403686523438, + "loss": 0.5658, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.245765209197998, + "rewards/margins": 0.9900045394897461, + "rewards/rejected": -3.235769271850586, + "step": 15030 + }, + { + "epoch": 0.98, + "learning_rate": 3.88110882172471e-09, + "logits/chosen": -1.0577032566070557, + "logits/rejected": -1.0624340772628784, + "logps/chosen": -433.12603759765625, + "logps/rejected": -525.9588012695312, + "loss": 0.5161, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.1242759227752686, + "rewards/margins": 0.9746935963630676, + "rewards/rejected": -3.0989696979522705, + "step": 15040 + }, + { + "epoch": 0.98, + "learning_rate": 3.569578250834371e-09, + "logits/chosen": -0.9758235812187195, + "logits/rejected": -1.0453417301177979, + "logps/chosen": -473.77239990234375, + "logps/rejected": -589.0740356445312, + "loss": 0.3695, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.7179462909698486, + "rewards/margins": 1.5306434631347656, + "rewards/rejected": -3.2485899925231934, + "step": 15050 + }, + { + "epoch": 0.99, + "learning_rate": 3.2710702839139353e-09, + "logits/chosen": -1.129165530204773, + "logits/rejected": -1.040967345237732, + "logps/chosen": -408.28240966796875, + "logps/rejected": -488.89935302734375, + "loss": 0.5936, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.036830425262451, + "rewards/margins": 0.7134383320808411, + "rewards/rejected": -2.7502689361572266, + "step": 15060 + }, + { + "epoch": 0.99, + "learning_rate": 2.9855864781272448e-09, + "logits/chosen": -1.214477300643921, + "logits/rejected": -1.325339674949646, + "logps/chosen": -396.2386169433594, + "logps/rejected": -524.412353515625, + "loss": 0.528, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.8456494808197021, + "rewards/margins": 0.9337970614433289, + "rewards/rejected": -2.779446601867676, + "step": 15070 + }, + { + "epoch": 0.99, + "learning_rate": 2.7131283226977665e-09, + "logits/chosen": -1.1110106706619263, + "logits/rejected": -1.1910868883132935, + "logps/chosen": -415.4817810058594, + "logps/rejected": -538.5015869140625, + "loss": 0.4812, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9603351354599, + "rewards/margins": 1.0916963815689087, + "rewards/rejected": -3.0520312786102295, + "step": 15080 + }, + { + "epoch": 0.99, + "learning_rate": 2.4536972389008205e-09, + "logits/chosen": -1.0061264038085938, + "logits/rejected": -0.9592840075492859, + "logps/chosen": -424.15185546875, + "logps/rejected": -516.12890625, + "loss": 0.4262, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.870168924331665, + "rewards/margins": 1.2966797351837158, + "rewards/rejected": -3.166848659515381, + "step": 15090 + }, + { + "epoch": 0.99, + "learning_rate": 2.20729458005553e-09, + "logits/chosen": -0.9116916656494141, + "logits/rejected": -0.6472299695014954, + "logps/chosen": -385.2953186035156, + "logps/rejected": -491.0865783691406, + "loss": 0.442, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9088608026504517, + "rewards/margins": 1.279929518699646, + "rewards/rejected": -3.1887905597686768, + "step": 15100 + }, + { + "epoch": 0.99, + "eval_logits/chosen": -1.0546156167984009, + "eval_logits/rejected": -0.9254134893417358, + "eval_logps/chosen": -432.0098571777344, + "eval_logps/rejected": -520.1915283203125, + "eval_loss": 0.4853096008300781, + "eval_rewards/accuracies": 0.671500027179718, + "eval_rewards/chosen": -2.0000486373901367, + "eval_rewards/margins": 1.0857477188110352, + "eval_rewards/rejected": -3.0857961177825928, + "eval_runtime": 732.0844, + "eval_samples_per_second": 2.732, + "eval_steps_per_second": 1.366, + "step": 15100 + }, + { + "epoch": 0.99, + "learning_rate": 1.9739216315192712e-09, + "logits/chosen": -1.0977634191513062, + "logits/rejected": -0.8465323448181152, + "logps/chosen": -429.33892822265625, + "logps/rejected": -495.4786682128906, + "loss": 0.5072, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0003013610839844, + "rewards/margins": 0.8983712196350098, + "rewards/rejected": -2.898672580718994, + "step": 15110 + }, + { + "epoch": 0.99, + "learning_rate": 1.7535796106796231e-09, + "logits/chosen": -1.1453430652618408, + "logits/rejected": -0.8692857623100281, + "logps/chosen": -476.8145446777344, + "logps/rejected": -496.99481201171875, + "loss": 0.4946, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.0127811431884766, + "rewards/margins": 1.0051124095916748, + "rewards/rejected": -3.0178937911987305, + "step": 15120 + }, + { + "epoch": 0.99, + "learning_rate": 1.5462696669482636e-09, + "logits/chosen": -1.240791916847229, + "logits/rejected": -1.0452940464019775, + "logps/chosen": -411.5857849121094, + "logps/rejected": -510.7543029785156, + "loss": 0.5611, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.903436303138733, + "rewards/margins": 0.9152453541755676, + "rewards/rejected": -2.8186817169189453, + "step": 15130 + }, + { + "epoch": 0.99, + "learning_rate": 1.3519928817556927e-09, + "logits/chosen": -1.0189902782440186, + "logits/rejected": -0.8792413473129272, + "logps/chosen": -352.0376281738281, + "logps/rejected": -434.4034729003906, + "loss": 0.5348, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.786441445350647, + "rewards/margins": 0.8181886672973633, + "rewards/rejected": -2.6046297550201416, + "step": 15140 + }, + { + "epoch": 0.99, + "learning_rate": 1.1707502685448512e-09, + "logits/chosen": -1.2458264827728271, + "logits/rejected": -1.0184621810913086, + "logps/chosen": -460.77850341796875, + "logps/rejected": -509.570068359375, + "loss": 0.5115, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -2.385084629058838, + "rewards/margins": 1.0698010921478271, + "rewards/rejected": -3.454885959625244, + "step": 15150 + }, + { + "epoch": 0.99, + "learning_rate": 1.002542772765569e-09, + "logits/chosen": -0.9062987565994263, + "logits/rejected": -0.7371143698692322, + "logps/chosen": -374.9570617675781, + "logps/rejected": -443.7781677246094, + "loss": 0.4719, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -1.863532304763794, + "rewards/margins": 1.0712873935699463, + "rewards/rejected": -2.9348196983337402, + "step": 15160 + }, + { + "epoch": 0.99, + "learning_rate": 8.473712718709559e-10, + "logits/chosen": -1.0819716453552246, + "logits/rejected": -1.0894006490707397, + "logps/chosen": -406.87353515625, + "logps/rejected": -448.4830017089844, + "loss": 0.5974, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -2.131286144256592, + "rewards/margins": 0.52387535572052, + "rewards/rejected": -2.6551613807678223, + "step": 15170 + }, + { + "epoch": 0.99, + "learning_rate": 7.052365753112966e-10, + "logits/chosen": -0.8242332339286804, + "logits/rejected": -0.8488165736198425, + "logps/chosen": -464.1226501464844, + "logps/rejected": -555.234130859375, + "loss": 0.5364, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.2880654335021973, + "rewards/margins": 1.098871111869812, + "rewards/rejected": -3.386936902999878, + "step": 15180 + }, + { + "epoch": 0.99, + "learning_rate": 5.761394245307195e-10, + "logits/chosen": -0.8750588297843933, + "logits/rejected": -0.9825660586357117, + "logps/chosen": -440.34344482421875, + "logps/rejected": -505.38800048828125, + "loss": 0.5564, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.0326123237609863, + "rewards/margins": 0.6625606417655945, + "rewards/rejected": -2.6951727867126465, + "step": 15190 + }, + { + "epoch": 0.99, + "learning_rate": 4.6008049296358826e-10, + "logits/chosen": -1.1716618537902832, + "logits/rejected": -1.0131809711456299, + "logps/chosen": -359.4949645996094, + "logps/rejected": -432.78948974609375, + "loss": 0.4814, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.734723687171936, + "rewards/margins": 1.0153820514678955, + "rewards/rejected": -2.750105619430542, + "step": 15200 + }, + { + "epoch": 0.99, + "eval_logits/chosen": -1.0581306219100952, + "eval_logits/rejected": -0.9285818338394165, + "eval_logps/chosen": -431.9893493652344, + "eval_logps/rejected": -520.1320190429688, + "eval_loss": 0.4854483902454376, + "eval_rewards/accuracies": 0.671999990940094, + "eval_rewards/chosen": -1.999843716621399, + "eval_rewards/margins": 1.0853569507598877, + "eval_rewards/rejected": -3.085200548171997, + "eval_runtime": 730.109, + "eval_samples_per_second": 2.739, + "eval_steps_per_second": 1.37, + "step": 15200 + }, + { + "epoch": 1.0, + "learning_rate": 3.5706038603006146e-10, + "logits/chosen": -1.0735647678375244, + "logits/rejected": -1.0667097568511963, + "logps/chosen": -445.7269592285156, + "logps/rejected": -559.1290283203125, + "loss": 0.4524, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.5611655712127686, + "rewards/margins": 1.276451826095581, + "rewards/rejected": -2.8376173973083496, + "step": 15210 + }, + { + "epoch": 1.0, + "learning_rate": 2.670796411333165e-10, + "logits/chosen": -1.4009530544281006, + "logits/rejected": -1.077488660812378, + "logps/chosen": -432.26788330078125, + "logps/rejected": -525.7315673828125, + "loss": 0.503, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -2.1245641708374023, + "rewards/margins": 1.0576976537704468, + "rewards/rejected": -3.1822619438171387, + "step": 15220 + }, + { + "epoch": 1.0, + "learning_rate": 1.9013872765677455e-10, + "logits/chosen": -1.1325209140777588, + "logits/rejected": -1.025263786315918, + "logps/chosen": -384.40570068359375, + "logps/rejected": -450.3807678222656, + "loss": 0.5252, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.64892578125, + "rewards/margins": 0.785096287727356, + "rewards/rejected": -2.4340221881866455, + "step": 15230 + }, + { + "epoch": 1.0, + "learning_rate": 1.262380469624347e-10, + "logits/chosen": -1.0579341650009155, + "logits/rejected": -0.881651759147644, + "logps/chosen": -395.76531982421875, + "logps/rejected": -468.01776123046875, + "loss": 0.4803, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -1.9175841808319092, + "rewards/margins": 0.9516212344169617, + "rewards/rejected": -2.8692054748535156, + "step": 15240 + }, + { + "epoch": 1.0, + "learning_rate": 7.53779323872661e-11, + "logits/chosen": -1.0222140550613403, + "logits/rejected": -0.9701115489006042, + "logps/chosen": -397.0771789550781, + "logps/rejected": -520.7330322265625, + "loss": 0.6077, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.052137851715088, + "rewards/margins": 1.1761195659637451, + "rewards/rejected": -3.228257417678833, + "step": 15250 + }, + { + "epoch": 1.0, + "learning_rate": 3.7558649242652734e-11, + "logits/chosen": -1.4560601711273193, + "logits/rejected": -1.136359453201294, + "logps/chosen": -625.4713134765625, + "logps/rejected": -639.3870239257812, + "loss": 0.5194, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -2.287875175476074, + "rewards/margins": 0.9348654747009277, + "rewards/rejected": -3.222740650177002, + "step": 15260 + }, + { + "epoch": 1.0, + "learning_rate": 1.2780394812450526e-11, + "logits/chosen": -0.8604592084884644, + "logits/rejected": -0.9040239453315735, + "logps/chosen": -468.42352294921875, + "logps/rejected": -587.9859619140625, + "loss": 0.4393, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -2.362797260284424, + "rewards/margins": 1.1964504718780518, + "rewards/rejected": -3.559248447418213, + "step": 15270 + }, + { + "epoch": 1.0, + "learning_rate": 1.0432983521546646e-12, + "logits/chosen": -0.9416975975036621, + "logits/rejected": -0.9361570477485657, + "logps/chosen": -362.9610595703125, + "logps/rejected": -484.21112060546875, + "loss": 0.4863, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -1.7949488162994385, + "rewards/margins": 1.024044394493103, + "rewards/rejected": -2.818993091583252, + "step": 15280 + }, + { + "epoch": 1.0, + "step": 15284, + "total_flos": 0.0, + "train_loss": 0.5233479982519362, + "train_runtime": 176392.38, + "train_samples_per_second": 0.347, + "train_steps_per_second": 0.087 + } + ], + "logging_steps": 10, + "max_steps": 15284, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 100, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}