diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,23868 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 100, + "global_step": 15284, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 3.270111183780249e-09, + "logits/chosen": -2.634561777114868, + "logits/rejected": -2.673060417175293, + "logps/chosen": -207.5323944091797, + "logps/rejected": -286.9266052246094, + "loss": 2500.0, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.0, + "learning_rate": 3.270111183780249e-08, + "logits/chosen": -2.2174882888793945, + "logits/rejected": -1.965146541595459, + "logps/chosen": -185.96807861328125, + "logps/rejected": -165.34136962890625, + "loss": 2514.281, + "rewards/accuracies": 0.3055555522441864, + "rewards/chosen": -0.0004766077909152955, + "rewards/margins": -0.0013973360182717443, + "rewards/rejected": 0.0009207282564602792, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 6.540222367560497e-08, + "logits/chosen": -2.4321837425231934, + "logits/rejected": -2.2228550910949707, + "logps/chosen": -232.39230346679688, + "logps/rejected": -231.34878540039062, + "loss": 2502.0375, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0005583365564234555, + "rewards/margins": -0.00018428356270305812, + "rewards/rejected": 0.0007426199736073613, + "step": 20 + }, + { + "epoch": 0.0, + "learning_rate": 9.810333551340746e-08, + "logits/chosen": -2.257960796356201, + "logits/rejected": -2.1621832847595215, + "logps/chosen": -197.38980102539062, + "logps/rejected": -219.08255004882812, + "loss": 2497.7605, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.00034341320861130953, + "rewards/margins": 0.0002548896591179073, + "rewards/rejected": 8.852362225297838e-05, + "step": 30 + }, + { + "epoch": 0.0, + "learning_rate": 1.3080444735120995e-07, + "logits/chosen": -2.212952136993408, + "logits/rejected": -2.2520880699157715, + "logps/chosen": -275.9777526855469, + "logps/rejected": -265.69256591796875, + "loss": 2506.1715, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": 9.906295599648729e-05, + "rewards/margins": -0.0005692678969353437, + "rewards/rejected": 0.0006683307001367211, + "step": 40 + }, + { + "epoch": 0.0, + "learning_rate": 1.6350555918901243e-07, + "logits/chosen": -2.3477938175201416, + "logits/rejected": -2.1409664154052734, + "logps/chosen": -204.83493041992188, + "logps/rejected": -184.77113342285156, + "loss": 2495.4912, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.0002936377131845802, + "rewards/margins": 0.0004673409857787192, + "rewards/rejected": -0.0007609786698594689, + "step": 50 + }, + { + "epoch": 0.0, + "learning_rate": 1.9620667102681492e-07, + "logits/chosen": -2.3064382076263428, + "logits/rejected": -2.067274570465088, + "logps/chosen": -209.7997589111328, + "logps/rejected": -185.90098571777344, + "loss": 2507.9305, + "rewards/accuracies": 0.375, + "rewards/chosen": 6.11677096458152e-05, + "rewards/margins": -0.0007640757248736918, + "rewards/rejected": 0.0008252434199675918, + "step": 60 + }, + { + "epoch": 0.0, + "learning_rate": 2.289077828646174e-07, + "logits/chosen": -2.2702558040618896, + "logits/rejected": -2.1572697162628174, + "logps/chosen": -217.9700927734375, + "logps/rejected": -207.92623901367188, + "loss": 2497.8785, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.00139452307485044, + "rewards/margins": 0.00025031311088241637, + "rewards/rejected": 0.001144210109487176, + "step": 70 + }, + { + "epoch": 0.01, + "learning_rate": 2.616088947024199e-07, + "logits/chosen": -2.508427381515503, + "logits/rejected": -2.2270090579986572, + "logps/chosen": -258.7547912597656, + "logps/rejected": -213.7139129638672, + "loss": 2487.6809, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.001956066582351923, + "rewards/margins": 0.0012630863348022103, + "rewards/rejected": 0.0006929802475497127, + "step": 80 + }, + { + "epoch": 0.01, + "learning_rate": 2.943100065402224e-07, + "logits/chosen": -2.258317708969116, + "logits/rejected": -2.173069715499878, + "logps/chosen": -184.67755126953125, + "logps/rejected": -165.4068603515625, + "loss": 2504.4389, + "rewards/accuracies": 0.32499998807907104, + "rewards/chosen": 0.0010724717285484076, + "rewards/margins": -0.00042183371260762215, + "rewards/rejected": 0.0014943054411560297, + "step": 90 + }, + { + "epoch": 0.01, + "learning_rate": 3.2701111837802487e-07, + "logits/chosen": -2.4314677715301514, + "logits/rejected": -2.4265189170837402, + "logps/chosen": -168.7687225341797, + "logps/rejected": -183.88238525390625, + "loss": 2504.3189, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": 0.0016615685308352113, + "rewards/margins": -0.00040531111881136894, + "rewards/rejected": 0.0020668795332312584, + "step": 100 + }, + { + "epoch": 0.01, + "eval_logits/chosen": -2.3479208946228027, + "eval_logits/rejected": -2.1595327854156494, + "eval_logps/chosen": -231.77784729003906, + "eval_logps/rejected": -211.47471618652344, + "eval_loss": 2491.39111328125, + "eval_rewards/accuracies": 0.49149999022483826, + "eval_rewards/chosen": 0.002271000761538744, + "eval_rewards/margins": 0.000899210455827415, + "eval_rewards/rejected": 0.0013717904221266508, + "eval_runtime": 723.3073, + "eval_samples_per_second": 2.765, + "eval_steps_per_second": 1.383, + "step": 100 + }, + { + "epoch": 0.01, + "learning_rate": 3.5971223021582736e-07, + "logits/chosen": -2.3472182750701904, + "logits/rejected": -1.9991645812988281, + "logps/chosen": -222.425048828125, + "logps/rejected": -167.09478759765625, + "loss": 2477.4027, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.00370473088696599, + "rewards/margins": 0.0023065346758812666, + "rewards/rejected": 0.0013981962110847235, + "step": 110 + }, + { + "epoch": 0.01, + "learning_rate": 3.9241334205362984e-07, + "logits/chosen": -2.3386950492858887, + "logits/rejected": -2.244011402130127, + "logps/chosen": -224.01455688476562, + "logps/rejected": -234.2311553955078, + "loss": 2486.7041, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": 0.002775652799755335, + "rewards/margins": 0.0013752636732533574, + "rewards/rejected": 0.001400388777256012, + "step": 120 + }, + { + "epoch": 0.01, + "learning_rate": 4.251144538914324e-07, + "logits/chosen": -2.261671543121338, + "logits/rejected": -2.2174248695373535, + "logps/chosen": -149.36570739746094, + "logps/rejected": -148.32887268066406, + "loss": 2481.2488, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": 0.0028014422859996557, + "rewards/margins": 0.0019072892609983683, + "rewards/rejected": 0.0008941531996242702, + "step": 130 + }, + { + "epoch": 0.01, + "learning_rate": 4.578155657292348e-07, + "logits/chosen": -2.322067975997925, + "logits/rejected": -2.2229113578796387, + "logps/chosen": -225.69760131835938, + "logps/rejected": -159.5988006591797, + "loss": 2483.5568, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.002613655524328351, + "rewards/margins": 0.0016905177617445588, + "rewards/rejected": 0.0009231379954144359, + "step": 140 + }, + { + "epoch": 0.01, + "learning_rate": 4.905166775670374e-07, + "logits/chosen": -2.3663547039031982, + "logits/rejected": -2.157618999481201, + "logps/chosen": -230.94027709960938, + "logps/rejected": -229.13720703125, + "loss": 2461.1934, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.0041537596844136715, + "rewards/margins": 0.003948424942791462, + "rewards/rejected": 0.00020533411588985473, + "step": 150 + }, + { + "epoch": 0.01, + "learning_rate": 5.232177894048398e-07, + "logits/chosen": -2.2142300605773926, + "logits/rejected": -2.2270874977111816, + "logps/chosen": -260.3035888671875, + "logps/rejected": -224.7972412109375, + "loss": 2465.09, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.005649039521813393, + "rewards/margins": 0.003626276273280382, + "rewards/rejected": 0.002022763481363654, + "step": 160 + }, + { + "epoch": 0.01, + "learning_rate": 5.559189012426422e-07, + "logits/chosen": -2.3138108253479004, + "logits/rejected": -2.0362300872802734, + "logps/chosen": -180.47308349609375, + "logps/rejected": -156.69332885742188, + "loss": 2483.0064, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0037951308768242598, + "rewards/margins": 0.0018231167923659086, + "rewards/rejected": 0.0019720138516277075, + "step": 170 + }, + { + "epoch": 0.01, + "learning_rate": 5.886200130804448e-07, + "logits/chosen": -2.397017240524292, + "logits/rejected": -2.3366546630859375, + "logps/chosen": -217.3719940185547, + "logps/rejected": -198.6235809326172, + "loss": 2434.8789, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.00856606476008892, + "rewards/margins": 0.006757465191185474, + "rewards/rejected": 0.0018085993360728025, + "step": 180 + }, + { + "epoch": 0.01, + "learning_rate": 6.213211249182473e-07, + "logits/chosen": -2.068624258041382, + "logits/rejected": -2.1739213466644287, + "logps/chosen": -190.7734832763672, + "logps/rejected": -208.3523406982422, + "loss": 2460.0418, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.007227701134979725, + "rewards/margins": 0.004224286414682865, + "rewards/rejected": 0.0030034154187887907, + "step": 190 + }, + { + "epoch": 0.01, + "learning_rate": 6.540222367560497e-07, + "logits/chosen": -2.275643825531006, + "logits/rejected": -2.2396295070648193, + "logps/chosen": -146.48544311523438, + "logps/rejected": -177.35043334960938, + "loss": 2448.3082, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.007266052067279816, + "rewards/margins": 0.005484951194375753, + "rewards/rejected": 0.001781100989319384, + "step": 200 + }, + { + "epoch": 0.01, + "eval_logits/chosen": -2.346447467803955, + "eval_logits/rejected": -2.158315420150757, + "eval_logps/chosen": -231.08497619628906, + "eval_logps/rejected": -211.30442810058594, + "eval_loss": 2442.388427734375, + "eval_rewards/accuracies": 0.5740000009536743, + "eval_rewards/chosen": 0.009199734777212143, + "eval_rewards/margins": 0.006124937906861305, + "eval_rewards/rejected": 0.0030747964046895504, + "eval_runtime": 722.3489, + "eval_samples_per_second": 2.769, + "eval_steps_per_second": 1.384, + "step": 200 + }, + { + "epoch": 0.01, + "learning_rate": 6.867233485938523e-07, + "logits/chosen": -2.425363302230835, + "logits/rejected": -2.2016208171844482, + "logps/chosen": -218.32437133789062, + "logps/rejected": -187.9815673828125, + "loss": 2401.2883, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.012091477401554585, + "rewards/margins": 0.010248173959553242, + "rewards/rejected": 0.0018433047225698829, + "step": 210 + }, + { + "epoch": 0.01, + "learning_rate": 7.194244604316547e-07, + "logits/chosen": -2.2120730876922607, + "logits/rejected": -2.03997802734375, + "logps/chosen": -182.62161254882812, + "logps/rejected": -174.7395782470703, + "loss": 2451.2725, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": 0.0071476297453045845, + "rewards/margins": 0.005110512487590313, + "rewards/rejected": 0.0020371167920529842, + "step": 220 + }, + { + "epoch": 0.02, + "learning_rate": 7.521255722694571e-07, + "logits/chosen": -2.4184441566467285, + "logits/rejected": -2.023744583129883, + "logps/chosen": -278.6354675292969, + "logps/rejected": -183.7732696533203, + "loss": 2448.2492, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": 0.011438937857747078, + "rewards/margins": 0.005931592546403408, + "rewards/rejected": 0.005507343914359808, + "step": 230 + }, + { + "epoch": 0.02, + "learning_rate": 7.848266841072597e-07, + "logits/chosen": -2.1973490715026855, + "logits/rejected": -2.162393808364868, + "logps/chosen": -214.4296112060547, + "logps/rejected": -206.10311889648438, + "loss": 2384.1496, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.014212613925337791, + "rewards/margins": 0.012409132905304432, + "rewards/rejected": 0.001803480088710785, + "step": 240 + }, + { + "epoch": 0.02, + "learning_rate": 8.175277959450622e-07, + "logits/chosen": -2.1652462482452393, + "logits/rejected": -2.323850154876709, + "logps/chosen": -218.1637420654297, + "logps/rejected": -220.7129364013672, + "loss": 2410.3398, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.022624235600233078, + "rewards/margins": 0.010092777200043201, + "rewards/rejected": 0.012531456537544727, + "step": 250 + }, + { + "epoch": 0.02, + "learning_rate": 8.502289077828648e-07, + "logits/chosen": -2.500765323638916, + "logits/rejected": -2.1402554512023926, + "logps/chosen": -253.96597290039062, + "logps/rejected": -189.17242431640625, + "loss": 2355.3189, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.031890373677015305, + "rewards/margins": 0.01565035805106163, + "rewards/rejected": 0.016240015625953674, + "step": 260 + }, + { + "epoch": 0.02, + "learning_rate": 8.829300196206672e-07, + "logits/chosen": -2.418590784072876, + "logits/rejected": -2.124760627746582, + "logps/chosen": -246.11233520507812, + "logps/rejected": -230.88671875, + "loss": 2316.4652, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.02698536589741707, + "rewards/margins": 0.020038802176713943, + "rewards/rejected": 0.0069465613923966885, + "step": 270 + }, + { + "epoch": 0.02, + "learning_rate": 9.156311314584696e-07, + "logits/chosen": -2.3023593425750732, + "logits/rejected": -2.1883482933044434, + "logps/chosen": -159.69656372070312, + "logps/rejected": -146.34173583984375, + "loss": 2402.9592, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": 0.024918871000409126, + "rewards/margins": 0.011144409887492657, + "rewards/rejected": 0.013774457387626171, + "step": 280 + }, + { + "epoch": 0.02, + "learning_rate": 9.483322432962722e-07, + "logits/chosen": -2.548877716064453, + "logits/rejected": -2.1554806232452393, + "logps/chosen": -281.9593200683594, + "logps/rejected": -225.94284057617188, + "loss": 2370.7814, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.032194942235946655, + "rewards/margins": 0.014771776273846626, + "rewards/rejected": 0.01742316409945488, + "step": 290 + }, + { + "epoch": 0.02, + "learning_rate": 9.810333551340747e-07, + "logits/chosen": -2.335076332092285, + "logits/rejected": -2.1912741661071777, + "logps/chosen": -265.0003356933594, + "logps/rejected": -239.0242156982422, + "loss": 2268.8053, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.03659301623702049, + "rewards/margins": 0.0267319492995739, + "rewards/rejected": 0.009861066937446594, + "step": 300 + }, + { + "epoch": 0.02, + "eval_logits/chosen": -2.3479902744293213, + "eval_logits/rejected": -2.159848690032959, + "eval_logps/chosen": -228.60084533691406, + "eval_logps/rejected": -210.1324005126953, + "eval_loss": 2338.618408203125, + "eval_rewards/accuracies": 0.5855000019073486, + "eval_rewards/chosen": 0.034041181206703186, + "eval_rewards/margins": 0.019246207550168037, + "eval_rewards/rejected": 0.014794973656535149, + "eval_runtime": 721.764, + "eval_samples_per_second": 2.771, + "eval_steps_per_second": 1.385, + "step": 300 + }, + { + "epoch": 0.02, + "learning_rate": 1.0137344669718771e-06, + "logits/chosen": -2.3462746143341064, + "logits/rejected": -2.360459804534912, + "logps/chosen": -167.21005249023438, + "logps/rejected": -156.17391967773438, + "loss": 2456.4926, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.02280285768210888, + "rewards/margins": 0.006144453771412373, + "rewards/rejected": 0.01665840484201908, + "step": 310 + }, + { + "epoch": 0.02, + "learning_rate": 1.0464355788096796e-06, + "logits/chosen": -2.4511752128601074, + "logits/rejected": -2.057394504547119, + "logps/chosen": -221.5251007080078, + "logps/rejected": -192.42202758789062, + "loss": 2365.6414, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.036400049924850464, + "rewards/margins": 0.016477955505251884, + "rewards/rejected": 0.01992209441959858, + "step": 320 + }, + { + "epoch": 0.02, + "learning_rate": 1.079136690647482e-06, + "logits/chosen": -2.431947708129883, + "logits/rejected": -2.1998298168182373, + "logps/chosen": -203.33926391601562, + "logps/rejected": -176.35411071777344, + "loss": 2227.7551, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.05072016641497612, + "rewards/margins": 0.031620416790246964, + "rewards/rejected": 0.019099745899438858, + "step": 330 + }, + { + "epoch": 0.02, + "learning_rate": 1.1118378024852844e-06, + "logits/chosen": -2.1818463802337646, + "logits/rejected": -2.3470516204833984, + "logps/chosen": -150.8397216796875, + "logps/rejected": -178.4027099609375, + "loss": 2532.8238, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.02874670922756195, + "rewards/margins": 0.0012015759712085128, + "rewards/rejected": 0.027545129880309105, + "step": 340 + }, + { + "epoch": 0.02, + "learning_rate": 1.144538914323087e-06, + "logits/chosen": -2.41914963722229, + "logits/rejected": -1.9812551736831665, + "logps/chosen": -317.7720031738281, + "logps/rejected": -248.2475128173828, + "loss": 2216.5986, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.04338701814413071, + "rewards/margins": 0.03322090953588486, + "rewards/rejected": 0.010166105814278126, + "step": 350 + }, + { + "epoch": 0.02, + "learning_rate": 1.1772400261608895e-06, + "logits/chosen": -2.4831156730651855, + "logits/rejected": -2.1900362968444824, + "logps/chosen": -219.9840850830078, + "logps/rejected": -193.16220092773438, + "loss": 2227.2625, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.054959487169981, + "rewards/margins": 0.03342561423778534, + "rewards/rejected": 0.021533865481615067, + "step": 360 + }, + { + "epoch": 0.02, + "learning_rate": 1.2099411379986922e-06, + "logits/chosen": -2.163857936859131, + "logits/rejected": -2.239915609359741, + "logps/chosen": -192.18533325195312, + "logps/rejected": -206.8373260498047, + "loss": 2365.8025, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.05311382934451103, + "rewards/margins": 0.01884276606142521, + "rewards/rejected": 0.03427106887102127, + "step": 370 + }, + { + "epoch": 0.02, + "learning_rate": 1.2426422498364946e-06, + "logits/chosen": -2.323566436767578, + "logits/rejected": -2.0360183715820312, + "logps/chosen": -216.2499237060547, + "logps/rejected": -162.70449829101562, + "loss": 2159.9684, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.053983062505722046, + "rewards/margins": 0.040319375693798065, + "rewards/rejected": 0.013663689605891705, + "step": 380 + }, + { + "epoch": 0.03, + "learning_rate": 1.2753433616742968e-06, + "logits/chosen": -2.321364164352417, + "logits/rejected": -2.241368293762207, + "logps/chosen": -181.35208129882812, + "logps/rejected": -245.04690551757812, + "loss": 2274.409, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.03948934003710747, + "rewards/margins": 0.03012676164507866, + "rewards/rejected": 0.009362581185996532, + "step": 390 + }, + { + "epoch": 0.03, + "learning_rate": 1.3080444735120995e-06, + "logits/chosen": -2.4815640449523926, + "logits/rejected": -2.1130177974700928, + "logps/chosen": -220.1060791015625, + "logps/rejected": -181.07186889648438, + "loss": 1992.3912, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.05386597663164139, + "rewards/margins": 0.06216004490852356, + "rewards/rejected": -0.008294071070849895, + "step": 400 + }, + { + "epoch": 0.03, + "eval_logits/chosen": -2.347727060317993, + "eval_logits/rejected": -2.159768581390381, + "eval_logps/chosen": -227.7647247314453, + "eval_logps/rejected": -211.16456604003906, + "eval_loss": 2218.60302734375, + "eval_rewards/accuracies": 0.6079999804496765, + "eval_rewards/chosen": 0.04240221157670021, + "eval_rewards/margins": 0.03792867437005043, + "eval_rewards/rejected": 0.004473535809665918, + "eval_runtime": 721.7154, + "eval_samples_per_second": 2.771, + "eval_steps_per_second": 1.386, + "step": 400 + }, + { + "epoch": 0.03, + "learning_rate": 1.3407455853499021e-06, + "logits/chosen": -2.464292049407959, + "logits/rejected": -2.315389394760132, + "logps/chosen": -257.93585205078125, + "logps/rejected": -226.0089111328125, + "loss": 2167.6412, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.0441744327545166, + "rewards/margins": 0.044694893062114716, + "rewards/rejected": -0.0005204584449529648, + "step": 410 + }, + { + "epoch": 0.03, + "learning_rate": 1.3734466971877046e-06, + "logits/chosen": -2.2925400733947754, + "logits/rejected": -2.19284725189209, + "logps/chosen": -177.65432739257812, + "logps/rejected": -174.91217041015625, + "loss": 2143.5434, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.04265324026346207, + "rewards/margins": 0.047265905886888504, + "rewards/rejected": -0.004612663760781288, + "step": 420 + }, + { + "epoch": 0.03, + "learning_rate": 1.406147809025507e-06, + "logits/chosen": -2.2861475944519043, + "logits/rejected": -2.099020004272461, + "logps/chosen": -210.7884521484375, + "logps/rejected": -183.57485961914062, + "loss": 2120.6955, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.04358814284205437, + "rewards/margins": 0.052858226001262665, + "rewards/rejected": -0.009270085021853447, + "step": 430 + }, + { + "epoch": 0.03, + "learning_rate": 1.4388489208633094e-06, + "logits/chosen": -2.385740041732788, + "logits/rejected": -2.1011345386505127, + "logps/chosen": -250.75271606445312, + "logps/rejected": -224.5830535888672, + "loss": 2251.0816, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.03744536638259888, + "rewards/margins": 0.037420663982629776, + "rewards/rejected": 2.4697743356227875e-05, + "step": 440 + }, + { + "epoch": 0.03, + "learning_rate": 1.471550032701112e-06, + "logits/chosen": -2.4100568294525146, + "logits/rejected": -2.207420825958252, + "logps/chosen": -184.5848388671875, + "logps/rejected": -194.29791259765625, + "loss": 2247.6176, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.05063049867749214, + "rewards/margins": 0.03911694884300232, + "rewards/rejected": 0.011513547971844673, + "step": 450 + }, + { + "epoch": 0.03, + "learning_rate": 1.5042511445389143e-06, + "logits/chosen": -2.157378673553467, + "logits/rejected": -2.204876184463501, + "logps/chosen": -148.00125122070312, + "logps/rejected": -214.1797332763672, + "loss": 2323.2939, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.029387522488832474, + "rewards/margins": 0.02752484381198883, + "rewards/rejected": 0.0018626749515533447, + "step": 460 + }, + { + "epoch": 0.03, + "learning_rate": 1.536952256376717e-06, + "logits/chosen": -2.049189329147339, + "logits/rejected": -2.111467123031616, + "logps/chosen": -200.1328582763672, + "logps/rejected": -251.7501678466797, + "loss": 2179.1457, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.00797753781080246, + "rewards/margins": 0.0605323500931263, + "rewards/rejected": -0.052554816007614136, + "step": 470 + }, + { + "epoch": 0.03, + "learning_rate": 1.5696533682145194e-06, + "logits/chosen": -2.4478366374969482, + "logits/rejected": -2.2150559425354004, + "logps/chosen": -184.00650024414062, + "logps/rejected": -158.0145263671875, + "loss": 2159.2445, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.002402497921139002, + "rewards/margins": 0.05066746473312378, + "rewards/rejected": -0.04826496168971062, + "step": 480 + }, + { + "epoch": 0.03, + "learning_rate": 1.602354480052322e-06, + "logits/chosen": -2.4213478565216064, + "logits/rejected": -2.331252336502075, + "logps/chosen": -264.4500732421875, + "logps/rejected": -215.86386108398438, + "loss": 2015.0939, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.021897155791521072, + "rewards/margins": 0.06365485489368439, + "rewards/rejected": -0.04175770282745361, + "step": 490 + }, + { + "epoch": 0.03, + "learning_rate": 1.6350555918901245e-06, + "logits/chosen": -2.2143654823303223, + "logits/rejected": -1.885271430015564, + "logps/chosen": -216.16024780273438, + "logps/rejected": -209.8026580810547, + "loss": 1894.6148, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.019088638946413994, + "rewards/margins": 0.08185932040214539, + "rewards/rejected": -0.06277067959308624, + "step": 500 + }, + { + "epoch": 0.03, + "eval_logits/chosen": -2.3530712127685547, + "eval_logits/rejected": -2.1646721363067627, + "eval_logps/chosen": -228.19320678710938, + "eval_logps/rejected": -214.5511932373047, + "eval_loss": 2099.700927734375, + "eval_rewards/accuracies": 0.609000027179718, + "eval_rewards/chosen": 0.0381174273788929, + "eval_rewards/margins": 0.06751034408807755, + "eval_rewards/rejected": -0.029392924159765244, + "eval_runtime": 723.2269, + "eval_samples_per_second": 2.765, + "eval_steps_per_second": 1.383, + "step": 500 + }, + { + "epoch": 0.03, + "learning_rate": 1.6677567037279269e-06, + "logits/chosen": -2.490424871444702, + "logits/rejected": -2.140373706817627, + "logps/chosen": -287.0405578613281, + "logps/rejected": -258.70330810546875, + "loss": 1731.6008, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.09223110973834991, + "rewards/margins": 0.12047766149044037, + "rewards/rejected": -0.028246542438864708, + "step": 510 + }, + { + "epoch": 0.03, + "learning_rate": 1.7004578155657295e-06, + "logits/chosen": -2.293846845626831, + "logits/rejected": -2.319566249847412, + "logps/chosen": -200.75526428222656, + "logps/rejected": -188.5205535888672, + "loss": 1976.1541, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.05964081361889839, + "rewards/margins": 0.08185950666666031, + "rewards/rejected": -0.022218704223632812, + "step": 520 + }, + { + "epoch": 0.03, + "learning_rate": 1.7331589274035318e-06, + "logits/chosen": -2.174065113067627, + "logits/rejected": -2.055107355117798, + "logps/chosen": -181.2405548095703, + "logps/rejected": -189.6827392578125, + "loss": 2011.5094, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.026860039681196213, + "rewards/margins": 0.08996396511793137, + "rewards/rejected": -0.06310392916202545, + "step": 530 + }, + { + "epoch": 0.04, + "learning_rate": 1.7658600392413344e-06, + "logits/chosen": -2.3945372104644775, + "logits/rejected": -2.235696315765381, + "logps/chosen": -202.3669891357422, + "logps/rejected": -195.96456909179688, + "loss": 1717.7557, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.004884188063442707, + "rewards/margins": 0.10967157781124115, + "rewards/rejected": -0.10478738695383072, + "step": 540 + }, + { + "epoch": 0.04, + "learning_rate": 1.7985611510791368e-06, + "logits/chosen": -2.39052152633667, + "logits/rejected": -1.9351263046264648, + "logps/chosen": -267.9992980957031, + "logps/rejected": -263.902099609375, + "loss": 1842.549, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.04449605196714401, + "rewards/margins": 0.12291286140680313, + "rewards/rejected": -0.16740891337394714, + "step": 550 + }, + { + "epoch": 0.04, + "learning_rate": 1.8312622629169393e-06, + "logits/chosen": -2.305115222930908, + "logits/rejected": -2.1655917167663574, + "logps/chosen": -278.2762145996094, + "logps/rejected": -246.7991180419922, + "loss": 2588.1338, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.1482178121805191, + "rewards/margins": 0.030435502529144287, + "rewards/rejected": -0.1786532998085022, + "step": 560 + }, + { + "epoch": 0.04, + "learning_rate": 1.8639633747547417e-06, + "logits/chosen": -2.3924612998962402, + "logits/rejected": -2.2275023460388184, + "logps/chosen": -246.13973999023438, + "logps/rejected": -217.94775390625, + "loss": 1937.843, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1699463427066803, + "rewards/margins": 0.07444851100444794, + "rewards/rejected": -0.24439485371112823, + "step": 570 + }, + { + "epoch": 0.04, + "learning_rate": 1.8966644865925443e-06, + "logits/chosen": -2.2448031902313232, + "logits/rejected": -2.270545482635498, + "logps/chosen": -259.0118103027344, + "logps/rejected": -274.3431701660156, + "loss": 2362.1373, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.19803020358085632, + "rewards/margins": 0.03936942294239998, + "rewards/rejected": -0.237399622797966, + "step": 580 + }, + { + "epoch": 0.04, + "learning_rate": 1.9293655984303466e-06, + "logits/chosen": -2.6608781814575195, + "logits/rejected": -2.240119457244873, + "logps/chosen": -299.0372009277344, + "logps/rejected": -222.8275604248047, + "loss": 2179.9811, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.160709410905838, + "rewards/margins": 0.06795064359903336, + "rewards/rejected": -0.22866006195545197, + "step": 590 + }, + { + "epoch": 0.04, + "learning_rate": 1.9620667102681494e-06, + "logits/chosen": -2.405942916870117, + "logits/rejected": -2.4169788360595703, + "logps/chosen": -203.57545471191406, + "logps/rejected": -193.13299560546875, + "loss": 2250.0029, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1432301253080368, + "rewards/margins": 0.03878119960427284, + "rewards/rejected": -0.18201133608818054, + "step": 600 + }, + { + "epoch": 0.04, + "eval_logits/chosen": -2.368892192840576, + "eval_logits/rejected": -2.1789205074310303, + "eval_logps/chosen": -246.8838653564453, + "eval_logps/rejected": -232.47830200195312, + "eval_loss": 2123.84716796875, + "eval_rewards/accuracies": 0.6100000143051147, + "eval_rewards/chosen": -0.148789182305336, + "eval_rewards/margins": 0.0598747693002224, + "eval_rewards/rejected": -0.2086639702320099, + "eval_runtime": 721.6425, + "eval_samples_per_second": 2.771, + "eval_steps_per_second": 1.386, + "step": 600 + }, + { + "epoch": 0.04, + "learning_rate": 1.994767822105952e-06, + "logits/chosen": -2.322364568710327, + "logits/rejected": -2.221156597137451, + "logps/chosen": -189.34378051757812, + "logps/rejected": -200.86306762695312, + "loss": 2275.9445, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": -0.14140108227729797, + "rewards/margins": 0.050842929631471634, + "rewards/rejected": -0.19224399328231812, + "step": 610 + }, + { + "epoch": 0.04, + "learning_rate": 2.0274689339437543e-06, + "logits/chosen": -2.2486069202423096, + "logits/rejected": -2.034440517425537, + "logps/chosen": -289.38165283203125, + "logps/rejected": -260.887939453125, + "loss": 2170.768, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10691903531551361, + "rewards/margins": 0.06860539317131042, + "rewards/rejected": -0.17552444338798523, + "step": 620 + }, + { + "epoch": 0.04, + "learning_rate": 2.0601700457815567e-06, + "logits/chosen": -2.3933961391448975, + "logits/rejected": -2.0247488021850586, + "logps/chosen": -274.1460876464844, + "logps/rejected": -234.80636596679688, + "loss": 2300.2445, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14791390299797058, + "rewards/margins": 0.04173479601740837, + "rewards/rejected": -0.18964870274066925, + "step": 630 + }, + { + "epoch": 0.04, + "learning_rate": 2.092871157619359e-06, + "logits/chosen": -2.4423108100891113, + "logits/rejected": -2.2551608085632324, + "logps/chosen": -188.18045043945312, + "logps/rejected": -207.5229949951172, + "loss": 2135.9152, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.11518283188343048, + "rewards/margins": 0.05631045252084732, + "rewards/rejected": -0.171493262052536, + "step": 640 + }, + { + "epoch": 0.04, + "learning_rate": 2.1255722694571616e-06, + "logits/chosen": -2.459855318069458, + "logits/rejected": -2.059840202331543, + "logps/chosen": -287.207275390625, + "logps/rejected": -203.07473754882812, + "loss": 1968.3957, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08312606811523438, + "rewards/margins": 0.0883508175611496, + "rewards/rejected": -0.17147688567638397, + "step": 650 + }, + { + "epoch": 0.04, + "learning_rate": 2.158273381294964e-06, + "logits/chosen": -2.366745710372925, + "logits/rejected": -2.262253999710083, + "logps/chosen": -221.7314910888672, + "logps/rejected": -228.86129760742188, + "loss": 2580.177, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.09304489940404892, + "rewards/margins": 0.016183609142899513, + "rewards/rejected": -0.10922850668430328, + "step": 660 + }, + { + "epoch": 0.04, + "learning_rate": 2.190974493132767e-06, + "logits/chosen": -2.373584270477295, + "logits/rejected": -2.0687379837036133, + "logps/chosen": -247.98617553710938, + "logps/rejected": -198.99374389648438, + "loss": 1987.7182, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10307125747203827, + "rewards/margins": 0.07264357805252075, + "rewards/rejected": -0.17571485042572021, + "step": 670 + }, + { + "epoch": 0.04, + "learning_rate": 2.223675604970569e-06, + "logits/chosen": -2.4493911266326904, + "logits/rejected": -2.242863178253174, + "logps/chosen": -219.24533081054688, + "logps/rejected": -210.0750274658203, + "loss": 2008.7635, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08511056005954742, + "rewards/margins": 0.08159992098808289, + "rewards/rejected": -0.16671046614646912, + "step": 680 + }, + { + "epoch": 0.05, + "learning_rate": 2.2563767168083718e-06, + "logits/chosen": -2.531895637512207, + "logits/rejected": -2.030471086502075, + "logps/chosen": -260.61981201171875, + "logps/rejected": -208.0525665283203, + "loss": 1935.4762, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.033680204302072525, + "rewards/margins": 0.10465041548013687, + "rewards/rejected": -0.1383306086063385, + "step": 690 + }, + { + "epoch": 0.05, + "learning_rate": 2.289077828646174e-06, + "logits/chosen": -2.364795684814453, + "logits/rejected": -2.1295928955078125, + "logps/chosen": -247.4524383544922, + "logps/rejected": -228.5208740234375, + "loss": 2168.8156, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.08391450345516205, + "rewards/margins": 0.05774035304784775, + "rewards/rejected": -0.1416548639535904, + "step": 700 + }, + { + "epoch": 0.05, + "eval_logits/chosen": -2.397294759750366, + "eval_logits/rejected": -2.2053017616271973, + "eval_logps/chosen": -235.78334045410156, + "eval_logps/rejected": -222.8819122314453, + "eval_loss": 2054.685546875, + "eval_rewards/accuracies": 0.6205000281333923, + "eval_rewards/chosen": -0.03778376802802086, + "eval_rewards/margins": 0.07491618394851685, + "eval_rewards/rejected": -0.112699955701828, + "eval_runtime": 722.0761, + "eval_samples_per_second": 2.77, + "eval_steps_per_second": 1.385, + "step": 700 + }, + { + "epoch": 0.05, + "learning_rate": 2.3217789404839766e-06, + "logits/chosen": -2.249499559402466, + "logits/rejected": -2.334920883178711, + "logps/chosen": -163.0601348876953, + "logps/rejected": -213.415283203125, + "loss": 2444.5898, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.06229011341929436, + "rewards/margins": 0.03412432223558426, + "rewards/rejected": -0.09641443192958832, + "step": 710 + }, + { + "epoch": 0.05, + "learning_rate": 2.354480052321779e-06, + "logits/chosen": -2.586360454559326, + "logits/rejected": -2.149442434310913, + "logps/chosen": -260.3407287597656, + "logps/rejected": -210.86669921875, + "loss": 1814.7633, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.019512873142957687, + "rewards/margins": 0.1016731709241867, + "rewards/rejected": -0.12118605524301529, + "step": 720 + }, + { + "epoch": 0.05, + "learning_rate": 2.3871811641595815e-06, + "logits/chosen": -2.410372257232666, + "logits/rejected": -2.2249979972839355, + "logps/chosen": -247.38467407226562, + "logps/rejected": -198.99746704101562, + "loss": 1741.6162, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.005922236945480108, + "rewards/margins": 0.10268674045801163, + "rewards/rejected": -0.10860898345708847, + "step": 730 + }, + { + "epoch": 0.05, + "learning_rate": 2.4198822759973843e-06, + "logits/chosen": -2.2442455291748047, + "logits/rejected": -2.2593936920166016, + "logps/chosen": -194.4607696533203, + "logps/rejected": -212.80746459960938, + "loss": 2339.5994, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.03854719549417496, + "rewards/margins": 0.04414154589176178, + "rewards/rejected": -0.08268874883651733, + "step": 740 + }, + { + "epoch": 0.05, + "learning_rate": 2.4525833878351864e-06, + "logits/chosen": -2.5059237480163574, + "logits/rejected": -2.302354574203491, + "logps/chosen": -254.85604858398438, + "logps/rejected": -196.12966918945312, + "loss": 2093.2186, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.020551379770040512, + "rewards/margins": 0.07553622126579285, + "rewards/rejected": -0.09608760476112366, + "step": 750 + }, + { + "epoch": 0.05, + "learning_rate": 2.4852844996729892e-06, + "logits/chosen": -2.2689011096954346, + "logits/rejected": -2.174787998199463, + "logps/chosen": -240.7899932861328, + "logps/rejected": -254.06173706054688, + "loss": 1940.8283, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.016656741499900818, + "rewards/margins": 0.11813600361347198, + "rewards/rejected": -0.1347927302122116, + "step": 760 + }, + { + "epoch": 0.05, + "learning_rate": 2.5179856115107916e-06, + "logits/chosen": -2.435136318206787, + "logits/rejected": -2.0990397930145264, + "logps/chosen": -273.72161865234375, + "logps/rejected": -228.80355834960938, + "loss": 1954.7613, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.002254198770970106, + "rewards/margins": 0.09030239284038544, + "rewards/rejected": -0.09255659580230713, + "step": 770 + }, + { + "epoch": 0.05, + "learning_rate": 2.5506867233485937e-06, + "logits/chosen": -2.5231940746307373, + "logits/rejected": -2.090528964996338, + "logps/chosen": -262.9032897949219, + "logps/rejected": -220.0452880859375, + "loss": 1513.9102, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 0.027926484122872353, + "rewards/margins": 0.14640609920024872, + "rewards/rejected": -0.11847962439060211, + "step": 780 + }, + { + "epoch": 0.05, + "learning_rate": 2.5833878351863965e-06, + "logits/chosen": -2.551626205444336, + "logits/rejected": -2.480762004852295, + "logps/chosen": -253.04141235351562, + "logps/rejected": -260.40289306640625, + "loss": 2008.6934, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.010434931144118309, + "rewards/margins": 0.09981861710548401, + "rewards/rejected": -0.11025355011224747, + "step": 790 + }, + { + "epoch": 0.05, + "learning_rate": 2.616088947024199e-06, + "logits/chosen": -2.2253010272979736, + "logits/rejected": -2.3965466022491455, + "logps/chosen": -202.80874633789062, + "logps/rejected": -232.88693237304688, + "loss": 2370.3576, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.06319282203912735, + "rewards/margins": 0.055518168956041336, + "rewards/rejected": -0.11871097981929779, + "step": 800 + }, + { + "epoch": 0.05, + "eval_logits/chosen": -2.426872968673706, + "eval_logits/rejected": -2.2322709560394287, + "eval_logps/chosen": -237.03123474121094, + "eval_logps/rejected": -226.0986785888672, + "eval_loss": 1997.0155029296875, + "eval_rewards/accuracies": 0.6140000224113464, + "eval_rewards/chosen": -0.050262872129678726, + "eval_rewards/margins": 0.0946047306060791, + "eval_rewards/rejected": -0.14486758410930634, + "eval_runtime": 722.8029, + "eval_samples_per_second": 2.767, + "eval_steps_per_second": 1.384, + "step": 800 + }, + { + "epoch": 0.05, + "learning_rate": 2.6487900588620014e-06, + "logits/chosen": -2.209472179412842, + "logits/rejected": -1.9674603939056396, + "logps/chosen": -203.85731506347656, + "logps/rejected": -167.39962768554688, + "loss": 2191.499, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.03741743415594101, + "rewards/margins": 0.05794162303209305, + "rewards/rejected": -0.09535904973745346, + "step": 810 + }, + { + "epoch": 0.05, + "learning_rate": 2.6814911706998042e-06, + "logits/chosen": -2.3804078102111816, + "logits/rejected": -2.196342945098877, + "logps/chosen": -226.281494140625, + "logps/rejected": -201.8038330078125, + "loss": 2224.7926, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.009422911331057549, + "rewards/margins": 0.06276295334100723, + "rewards/rejected": -0.07218586653470993, + "step": 820 + }, + { + "epoch": 0.05, + "learning_rate": 2.7141922825376067e-06, + "logits/chosen": -2.267695665359497, + "logits/rejected": -2.2699947357177734, + "logps/chosen": -261.9485168457031, + "logps/rejected": -283.9317626953125, + "loss": 1690.7559, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.040243808180093765, + "rewards/margins": 0.12052266299724579, + "rewards/rejected": -0.16076649725437164, + "step": 830 + }, + { + "epoch": 0.05, + "learning_rate": 2.746893394375409e-06, + "logits/chosen": -2.3985633850097656, + "logits/rejected": -2.2916035652160645, + "logps/chosen": -238.35885620117188, + "logps/rejected": -246.00900268554688, + "loss": 2056.7525, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.03964471071958542, + "rewards/margins": 0.09431429952383041, + "rewards/rejected": -0.13395901024341583, + "step": 840 + }, + { + "epoch": 0.06, + "learning_rate": 2.779594506213211e-06, + "logits/chosen": -2.320394992828369, + "logits/rejected": -2.0796637535095215, + "logps/chosen": -197.7746124267578, + "logps/rejected": -204.3472900390625, + "loss": 2249.3328, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.0791572779417038, + "rewards/margins": 0.06885950267314911, + "rewards/rejected": -0.1480167806148529, + "step": 850 + }, + { + "epoch": 0.06, + "learning_rate": 2.812295618051014e-06, + "logits/chosen": -2.4867045879364014, + "logits/rejected": -2.3241653442382812, + "logps/chosen": -275.2378234863281, + "logps/rejected": -225.60928344726562, + "loss": 1811.7268, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.05723363161087036, + "rewards/margins": 0.12098623812198639, + "rewards/rejected": -0.06375260651111603, + "step": 860 + }, + { + "epoch": 0.06, + "learning_rate": 2.8449967298888164e-06, + "logits/chosen": -2.4043803215026855, + "logits/rejected": -2.1745409965515137, + "logps/chosen": -176.84385681152344, + "logps/rejected": -154.92471313476562, + "loss": 2119.0828, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.00019865883223246783, + "rewards/margins": 0.061392270028591156, + "rewards/rejected": -0.061590928584337234, + "step": 870 + }, + { + "epoch": 0.06, + "learning_rate": 2.877697841726619e-06, + "logits/chosen": -2.478651285171509, + "logits/rejected": -2.372340679168701, + "logps/chosen": -216.61239624023438, + "logps/rejected": -206.92984008789062, + "loss": 2108.6172, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.022234294563531876, + "rewards/margins": 0.07132290303707123, + "rewards/rejected": -0.049088604748249054, + "step": 880 + }, + { + "epoch": 0.06, + "learning_rate": 2.9103989535644217e-06, + "logits/chosen": -2.387673854827881, + "logits/rejected": -2.503478765487671, + "logps/chosen": -216.41189575195312, + "logps/rejected": -253.4788818359375, + "loss": 2194.2061, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.00928253959864378, + "rewards/margins": 0.06536583602428436, + "rewards/rejected": -0.07464838773012161, + "step": 890 + }, + { + "epoch": 0.06, + "learning_rate": 2.943100065402224e-06, + "logits/chosen": -2.428986072540283, + "logits/rejected": -2.278474807739258, + "logps/chosen": -293.5047302246094, + "logps/rejected": -284.4921875, + "loss": 2219.9355, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.008615334518253803, + "rewards/margins": 0.0630418211221695, + "rewards/rejected": -0.07165715098381042, + "step": 900 + }, + { + "epoch": 0.06, + "eval_logits/chosen": -2.4120709896087646, + "eval_logits/rejected": -2.220082998275757, + "eval_logps/chosen": -229.2795867919922, + "eval_logps/rejected": -217.4352569580078, + "eval_loss": 2038.843994140625, + "eval_rewards/accuracies": 0.6200000047683716, + "eval_rewards/chosen": 0.02725347876548767, + "eval_rewards/margins": 0.08548705279827118, + "eval_rewards/rejected": -0.058233581483364105, + "eval_runtime": 724.9078, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.379, + "step": 900 + }, + { + "epoch": 0.06, + "learning_rate": 2.9758011772400266e-06, + "logits/chosen": -2.322211265563965, + "logits/rejected": -2.34004282951355, + "logps/chosen": -253.59512329101562, + "logps/rejected": -267.9804382324219, + "loss": 2021.3826, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.03714027255773544, + "rewards/margins": 0.07010693103075027, + "rewards/rejected": -0.03296665847301483, + "step": 910 + }, + { + "epoch": 0.06, + "learning_rate": 3.0085022890778286e-06, + "logits/chosen": -2.3901853561401367, + "logits/rejected": -2.1437230110168457, + "logps/chosen": -174.85421752929688, + "logps/rejected": -155.4096221923828, + "loss": 2090.9529, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.008853385224938393, + "rewards/margins": 0.07478860765695572, + "rewards/rejected": -0.06593521684408188, + "step": 920 + }, + { + "epoch": 0.06, + "learning_rate": 3.0412034009156314e-06, + "logits/chosen": -2.3286807537078857, + "logits/rejected": -2.497436285018921, + "logps/chosen": -246.6853485107422, + "logps/rejected": -249.7530975341797, + "loss": 2147.3672, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.109990693628788, + "rewards/margins": 0.0732945054769516, + "rewards/rejected": -0.183285191655159, + "step": 930 + }, + { + "epoch": 0.06, + "learning_rate": 3.073904512753434e-06, + "logits/chosen": -2.501011610031128, + "logits/rejected": -2.1241161823272705, + "logps/chosen": -247.611083984375, + "logps/rejected": -225.5044708251953, + "loss": 1653.3525, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17962466180324554, + "rewards/margins": 0.12821651995182037, + "rewards/rejected": -0.30784112215042114, + "step": 940 + }, + { + "epoch": 0.06, + "learning_rate": 3.1066056245912363e-06, + "logits/chosen": -2.373318672180176, + "logits/rejected": -2.4368274211883545, + "logps/chosen": -256.42279052734375, + "logps/rejected": -234.74411010742188, + "loss": 1920.7047, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.25438791513442993, + "rewards/margins": 0.10280318558216095, + "rewards/rejected": -0.3571911156177521, + "step": 950 + }, + { + "epoch": 0.06, + "learning_rate": 3.1393067364290387e-06, + "logits/chosen": -2.382197618484497, + "logits/rejected": -2.1782593727111816, + "logps/chosen": -240.29061889648438, + "logps/rejected": -214.4322509765625, + "loss": 1933.4619, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15163874626159668, + "rewards/margins": 0.12727205455303192, + "rewards/rejected": -0.2789107859134674, + "step": 960 + }, + { + "epoch": 0.06, + "learning_rate": 3.1720078482668416e-06, + "logits/chosen": -2.3237643241882324, + "logits/rejected": -2.1025829315185547, + "logps/chosen": -224.8642120361328, + "logps/rejected": -191.6042938232422, + "loss": 1851.0059, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13293561339378357, + "rewards/margins": 0.1384698450565338, + "rewards/rejected": -0.27140548825263977, + "step": 970 + }, + { + "epoch": 0.06, + "learning_rate": 3.204708960104644e-06, + "logits/chosen": -2.436340808868408, + "logits/rejected": -2.092273473739624, + "logps/chosen": -223.2788848876953, + "logps/rejected": -208.0653076171875, + "loss": 1677.9363, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.05876126140356064, + "rewards/margins": 0.17374159395694733, + "rewards/rejected": -0.23250284790992737, + "step": 980 + }, + { + "epoch": 0.06, + "learning_rate": 3.237410071942446e-06, + "logits/chosen": -2.4342200756073, + "logits/rejected": -2.1691014766693115, + "logps/chosen": -215.2308349609375, + "logps/rejected": -184.5440673828125, + "loss": 2402.4279, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08869779109954834, + "rewards/margins": 0.02877037599682808, + "rewards/rejected": -0.11746816337108612, + "step": 990 + }, + { + "epoch": 0.07, + "learning_rate": 3.270111183780249e-06, + "logits/chosen": -2.343592405319214, + "logits/rejected": -2.1235079765319824, + "logps/chosen": -205.7447509765625, + "logps/rejected": -195.60147094726562, + "loss": 2382.9531, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.05721202492713928, + "rewards/margins": 0.07859710603952408, + "rewards/rejected": -0.13580910861492157, + "step": 1000 + }, + { + "epoch": 0.07, + "eval_logits/chosen": -2.370666027069092, + "eval_logits/rejected": -2.1831679344177246, + "eval_logps/chosen": -232.2192840576172, + "eval_logps/rejected": -221.19912719726562, + "eval_loss": 2008.1322021484375, + "eval_rewards/accuracies": 0.6215000152587891, + "eval_rewards/chosen": -0.0021434456575661898, + "eval_rewards/margins": 0.09372860938310623, + "eval_rewards/rejected": -0.09587204456329346, + "eval_runtime": 725.0382, + "eval_samples_per_second": 2.758, + "eval_steps_per_second": 1.379, + "step": 1000 + }, + { + "epoch": 0.07, + "learning_rate": 3.3028122956180513e-06, + "logits/chosen": -2.2390599250793457, + "logits/rejected": -2.2753896713256836, + "logps/chosen": -210.17648315429688, + "logps/rejected": -244.66043090820312, + "loss": 2124.2566, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.019214505329728127, + "rewards/margins": 0.07015024125576019, + "rewards/rejected": -0.05093574523925781, + "step": 1010 + }, + { + "epoch": 0.07, + "learning_rate": 3.3355134074558538e-06, + "logits/chosen": -2.24173641204834, + "logits/rejected": -2.1411194801330566, + "logps/chosen": -221.272705078125, + "logps/rejected": -204.73448181152344, + "loss": 2000.4838, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.005399542860686779, + "rewards/margins": 0.09049306809902191, + "rewards/rejected": -0.09589260816574097, + "step": 1020 + }, + { + "epoch": 0.07, + "learning_rate": 3.368214519293656e-06, + "logits/chosen": -2.191763162612915, + "logits/rejected": -1.9984468221664429, + "logps/chosen": -196.1796417236328, + "logps/rejected": -185.42257690429688, + "loss": 1851.1479, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.027906540781259537, + "rewards/margins": 0.10141406208276749, + "rewards/rejected": -0.07350752502679825, + "step": 1030 + }, + { + "epoch": 0.07, + "learning_rate": 3.400915631131459e-06, + "logits/chosen": -2.1586947441101074, + "logits/rejected": -2.190293788909912, + "logps/chosen": -200.91342163085938, + "logps/rejected": -240.82846069335938, + "loss": 1875.4949, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.029409628361463547, + "rewards/margins": 0.11263708025217056, + "rewards/rejected": -0.142046719789505, + "step": 1040 + }, + { + "epoch": 0.07, + "learning_rate": 3.4336167429692615e-06, + "logits/chosen": -2.405604839324951, + "logits/rejected": -2.1920061111450195, + "logps/chosen": -215.37515258789062, + "logps/rejected": -208.7307891845703, + "loss": 1940.4701, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.008247396908700466, + "rewards/margins": 0.09609442949295044, + "rewards/rejected": -0.10434182733297348, + "step": 1050 + }, + { + "epoch": 0.07, + "learning_rate": 3.4663178548070635e-06, + "logits/chosen": -2.2052693367004395, + "logits/rejected": -2.268381357192993, + "logps/chosen": -213.3984375, + "logps/rejected": -206.3799591064453, + "loss": 1942.6387, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.015555182471871376, + "rewards/margins": 0.09423165023326874, + "rewards/rejected": -0.07867647707462311, + "step": 1060 + }, + { + "epoch": 0.07, + "learning_rate": 3.499018966644866e-06, + "logits/chosen": -2.344458818435669, + "logits/rejected": -2.147507905960083, + "logps/chosen": -193.80804443359375, + "logps/rejected": -196.10009765625, + "loss": 2087.9314, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.019830048084259033, + "rewards/margins": 0.09104426950216293, + "rewards/rejected": -0.11087431758642197, + "step": 1070 + }, + { + "epoch": 0.07, + "learning_rate": 3.531720078482669e-06, + "logits/chosen": -2.2609543800354004, + "logits/rejected": -1.932922124862671, + "logps/chosen": -232.9537353515625, + "logps/rejected": -231.40316772460938, + "loss": 2026.6357, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.07084006816148758, + "rewards/margins": 0.09680701792240143, + "rewards/rejected": -0.1676470935344696, + "step": 1080 + }, + { + "epoch": 0.07, + "learning_rate": 3.5644211903204712e-06, + "logits/chosen": -2.2446954250335693, + "logits/rejected": -2.2965035438537598, + "logps/chosen": -212.11300659179688, + "logps/rejected": -223.2448272705078, + "loss": 1987.2455, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13230548799037933, + "rewards/margins": 0.08318330347537994, + "rewards/rejected": -0.2154887616634369, + "step": 1090 + }, + { + "epoch": 0.07, + "learning_rate": 3.5971223021582737e-06, + "logits/chosen": -2.4021565914154053, + "logits/rejected": -2.26518177986145, + "logps/chosen": -273.38848876953125, + "logps/rejected": -225.50161743164062, + "loss": 2155.5914, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.17587730288505554, + "rewards/margins": 0.09228373318910599, + "rewards/rejected": -0.26816102862358093, + "step": 1100 + }, + { + "epoch": 0.07, + "eval_logits/chosen": -2.2892866134643555, + "eval_logits/rejected": -2.106736898422241, + "eval_logps/chosen": -252.53115844726562, + "eval_logps/rejected": -241.62571716308594, + "eval_loss": 2003.248046875, + "eval_rewards/accuracies": 0.6244999766349792, + "eval_rewards/chosen": -0.20526205003261566, + "eval_rewards/margins": 0.09487606585025787, + "eval_rewards/rejected": -0.30013811588287354, + "eval_runtime": 725.6609, + "eval_samples_per_second": 2.756, + "eval_steps_per_second": 1.378, + "step": 1100 + }, + { + "epoch": 0.07, + "learning_rate": 3.6298234139960765e-06, + "logits/chosen": -2.367889165878296, + "logits/rejected": -2.0205140113830566, + "logps/chosen": -238.40225219726562, + "logps/rejected": -195.3968048095703, + "loss": 2193.3691, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.20633168518543243, + "rewards/margins": 0.06665893644094467, + "rewards/rejected": -0.2729905843734741, + "step": 1110 + }, + { + "epoch": 0.07, + "learning_rate": 3.6625245258338785e-06, + "logits/chosen": -2.1692986488342285, + "logits/rejected": -2.098557949066162, + "logps/chosen": -257.96173095703125, + "logps/rejected": -344.1214294433594, + "loss": 1914.0918, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.19183330237865448, + "rewards/margins": 0.09699074178934097, + "rewards/rejected": -0.28882405161857605, + "step": 1120 + }, + { + "epoch": 0.07, + "learning_rate": 3.695225637671681e-06, + "logits/chosen": -2.4231133460998535, + "logits/rejected": -2.2229857444763184, + "logps/chosen": -227.0390625, + "logps/rejected": -200.2311553955078, + "loss": 2141.0023, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.19762437045574188, + "rewards/margins": 0.06268110871315002, + "rewards/rejected": -0.2603054940700531, + "step": 1130 + }, + { + "epoch": 0.07, + "learning_rate": 3.7279267495094834e-06, + "logits/chosen": -2.34079909324646, + "logits/rejected": -2.2142493724823, + "logps/chosen": -173.04766845703125, + "logps/rejected": -194.06484985351562, + "loss": 1875.1215, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1262076050043106, + "rewards/margins": 0.10891900211572647, + "rewards/rejected": -0.2351265847682953, + "step": 1140 + }, + { + "epoch": 0.08, + "learning_rate": 3.7606278613472863e-06, + "logits/chosen": -2.3947463035583496, + "logits/rejected": -2.054185628890991, + "logps/chosen": -288.93695068359375, + "logps/rejected": -225.27359008789062, + "loss": 1889.7932, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.06665666401386261, + "rewards/margins": 0.139047309756279, + "rewards/rejected": -0.205704003572464, + "step": 1150 + }, + { + "epoch": 0.08, + "learning_rate": 3.7933289731850887e-06, + "logits/chosen": -2.2144577503204346, + "logits/rejected": -1.9980432987213135, + "logps/chosen": -229.6927490234375, + "logps/rejected": -215.7831573486328, + "loss": 1700.6029, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.011321134865283966, + "rewards/margins": 0.14348730444908142, + "rewards/rejected": -0.1548084318637848, + "step": 1160 + }, + { + "epoch": 0.08, + "learning_rate": 3.826030085022891e-06, + "logits/chosen": -2.375175952911377, + "logits/rejected": -2.146986484527588, + "logps/chosen": -247.9508056640625, + "logps/rejected": -220.3432159423828, + "loss": 2110.4133, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.003009948180988431, + "rewards/margins": 0.08544488996267319, + "rewards/rejected": -0.08845484256744385, + "step": 1170 + }, + { + "epoch": 0.08, + "learning_rate": 3.858731196860693e-06, + "logits/chosen": -2.422227382659912, + "logits/rejected": -1.972865343093872, + "logps/chosen": -248.3476104736328, + "logps/rejected": -211.3714599609375, + "loss": 2249.0883, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.024858513846993446, + "rewards/margins": 0.05259479209780693, + "rewards/rejected": -0.02773628570139408, + "step": 1180 + }, + { + "epoch": 0.08, + "learning_rate": 3.891432308698496e-06, + "logits/chosen": -2.1258654594421387, + "logits/rejected": -2.0728182792663574, + "logps/chosen": -191.03858947753906, + "logps/rejected": -224.00015258789062, + "loss": 2100.7717, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.04191805049777031, + "rewards/margins": 0.0771632045507431, + "rewards/rejected": -0.03524515777826309, + "step": 1190 + }, + { + "epoch": 0.08, + "learning_rate": 3.924133420536299e-06, + "logits/chosen": -2.238049030303955, + "logits/rejected": -2.0524375438690186, + "logps/chosen": -180.01011657714844, + "logps/rejected": -153.81248474121094, + "loss": 2408.5309, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.02134573832154274, + "rewards/margins": 0.07205002009868622, + "rewards/rejected": -0.050704289227724075, + "step": 1200 + }, + { + "epoch": 0.08, + "eval_logits/chosen": -2.3281822204589844, + "eval_logits/rejected": -2.142608880996704, + "eval_logps/chosen": -228.13014221191406, + "eval_logps/rejected": -217.15615844726562, + "eval_loss": 1978.77734375, + "eval_rewards/accuracies": 0.640500009059906, + "eval_rewards/chosen": 0.038747940212488174, + "eval_rewards/margins": 0.09419067949056625, + "eval_rewards/rejected": -0.055442746728658676, + "eval_runtime": 728.1646, + "eval_samples_per_second": 2.747, + "eval_steps_per_second": 1.373, + "step": 1200 + }, + { + "epoch": 0.08, + "learning_rate": 3.956834532374101e-06, + "logits/chosen": -2.434722661972046, + "logits/rejected": -2.054490566253662, + "logps/chosen": -200.15614318847656, + "logps/rejected": -170.80064392089844, + "loss": 1546.1736, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.051065344363451004, + "rewards/margins": 0.14595820009708405, + "rewards/rejected": -0.09489286690950394, + "step": 1210 + }, + { + "epoch": 0.08, + "learning_rate": 3.989535644211904e-06, + "logits/chosen": -2.2990026473999023, + "logits/rejected": -2.0386624336242676, + "logps/chosen": -221.647705078125, + "logps/rejected": -209.01327514648438, + "loss": 2074.093, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.04520409181714058, + "rewards/margins": 0.0998666062951088, + "rewards/rejected": -0.14507070183753967, + "step": 1220 + }, + { + "epoch": 0.08, + "learning_rate": 4.022236756049706e-06, + "logits/chosen": -2.4798145294189453, + "logits/rejected": -2.0929503440856934, + "logps/chosen": -281.10369873046875, + "logps/rejected": -259.5100402832031, + "loss": 2039.6855, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.039665382355451584, + "rewards/margins": 0.10881809145212173, + "rewards/rejected": -0.1484834849834442, + "step": 1230 + }, + { + "epoch": 0.08, + "learning_rate": 4.054937867887509e-06, + "logits/chosen": -2.3605129718780518, + "logits/rejected": -1.9098589420318604, + "logps/chosen": -224.52041625976562, + "logps/rejected": -230.14990234375, + "loss": 1669.4992, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.002536333166062832, + "rewards/margins": 0.16417892277240753, + "rewards/rejected": -0.16671524941921234, + "step": 1240 + }, + { + "epoch": 0.08, + "learning_rate": 4.087638979725311e-06, + "logits/chosen": -2.280735492706299, + "logits/rejected": -2.3474297523498535, + "logps/chosen": -223.4026336669922, + "logps/rejected": -233.1063690185547, + "loss": 2110.8816, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.04437154531478882, + "rewards/margins": 0.08248762786388397, + "rewards/rejected": -0.03811608999967575, + "step": 1250 + }, + { + "epoch": 0.08, + "learning_rate": 4.1203400915631135e-06, + "logits/chosen": -2.1865649223327637, + "logits/rejected": -2.1238932609558105, + "logps/chosen": -233.7169189453125, + "logps/rejected": -214.2218017578125, + "loss": 2116.418, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.03614087775349617, + "rewards/margins": 0.0878569632768631, + "rewards/rejected": -0.05171608179807663, + "step": 1260 + }, + { + "epoch": 0.08, + "learning_rate": 4.153041203400916e-06, + "logits/chosen": -2.4024250507354736, + "logits/rejected": -2.246831178665161, + "logps/chosen": -256.627685546875, + "logps/rejected": -238.527099609375, + "loss": 2156.5352, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.0779462456703186, + "rewards/margins": 0.06534029543399811, + "rewards/rejected": 0.012605957686901093, + "step": 1270 + }, + { + "epoch": 0.08, + "learning_rate": 4.185742315238718e-06, + "logits/chosen": -2.307811737060547, + "logits/rejected": -2.0606985092163086, + "logps/chosen": -194.02505493164062, + "logps/rejected": -188.58154296875, + "loss": 2189.1459, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.04076526314020157, + "rewards/margins": 0.06423444300889969, + "rewards/rejected": -0.02346917614340782, + "step": 1280 + }, + { + "epoch": 0.08, + "learning_rate": 4.218443427076521e-06, + "logits/chosen": -2.2853074073791504, + "logits/rejected": -2.1252546310424805, + "logps/chosen": -181.89572143554688, + "logps/rejected": -189.04368591308594, + "loss": 2332.007, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.008628499694168568, + "rewards/margins": 0.05676931142807007, + "rewards/rejected": -0.048140816390514374, + "step": 1290 + }, + { + "epoch": 0.09, + "learning_rate": 4.251144538914323e-06, + "logits/chosen": -2.245187997817993, + "logits/rejected": -2.1047472953796387, + "logps/chosen": -223.224853515625, + "logps/rejected": -216.79763793945312, + "loss": 2340.302, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.026345735415816307, + "rewards/margins": 0.06768472492694855, + "rewards/rejected": -0.0940304696559906, + "step": 1300 + }, + { + "epoch": 0.09, + "eval_logits/chosen": -2.355750799179077, + "eval_logits/rejected": -2.1680800914764404, + "eval_logps/chosen": -228.8513641357422, + "eval_logps/rejected": -217.4835205078125, + "eval_loss": 1959.7281494140625, + "eval_rewards/accuracies": 0.6445000171661377, + "eval_rewards/chosen": 0.031535882502794266, + "eval_rewards/margins": 0.09025205671787262, + "eval_rewards/rejected": -0.058716174215078354, + "eval_runtime": 725.6305, + "eval_samples_per_second": 2.756, + "eval_steps_per_second": 1.378, + "step": 1300 + }, + { + "epoch": 0.09, + "learning_rate": 4.283845650752126e-06, + "logits/chosen": -2.388261079788208, + "logits/rejected": -2.1793575286865234, + "logps/chosen": -282.85009765625, + "logps/rejected": -228.8882293701172, + "loss": 1980.2102, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.03361871466040611, + "rewards/margins": 0.10229597240686417, + "rewards/rejected": -0.06867726892232895, + "step": 1310 + }, + { + "epoch": 0.09, + "learning_rate": 4.316546762589928e-06, + "logits/chosen": -2.3317654132843018, + "logits/rejected": -2.24820876121521, + "logps/chosen": -204.1890411376953, + "logps/rejected": -178.5457763671875, + "loss": 1923.8402, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.03738489747047424, + "rewards/margins": 0.0865490585565567, + "rewards/rejected": -0.04916415363550186, + "step": 1320 + }, + { + "epoch": 0.09, + "learning_rate": 4.349247874427731e-06, + "logits/chosen": -2.3867275714874268, + "logits/rejected": -2.0918281078338623, + "logps/chosen": -224.2469482421875, + "logps/rejected": -274.74603271484375, + "loss": 2061.5996, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.022870704531669617, + "rewards/margins": 0.10416515171527863, + "rewards/rejected": -0.08129443228244781, + "step": 1330 + }, + { + "epoch": 0.09, + "learning_rate": 4.381948986265534e-06, + "logits/chosen": -2.535256862640381, + "logits/rejected": -2.2696471214294434, + "logps/chosen": -263.3232727050781, + "logps/rejected": -261.365966796875, + "loss": 1796.1584, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.004785762168467045, + "rewards/margins": 0.11049681901931763, + "rewards/rejected": -0.1152825802564621, + "step": 1340 + }, + { + "epoch": 0.09, + "learning_rate": 4.414650098103336e-06, + "logits/chosen": -2.6866261959075928, + "logits/rejected": -2.4202332496643066, + "logps/chosen": -257.3060607910156, + "logps/rejected": -244.3971405029297, + "loss": 1999.8973, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.046464331448078156, + "rewards/margins": 0.09515171498060226, + "rewards/rejected": -0.04868737980723381, + "step": 1350 + }, + { + "epoch": 0.09, + "learning_rate": 4.447351209941138e-06, + "logits/chosen": -2.4166512489318848, + "logits/rejected": -2.1275861263275146, + "logps/chosen": -217.0367431640625, + "logps/rejected": -207.73403930664062, + "loss": 2038.8555, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07372106611728668, + "rewards/margins": 0.08838832378387451, + "rewards/rejected": -0.162109375, + "step": 1360 + }, + { + "epoch": 0.09, + "learning_rate": 4.480052321778941e-06, + "logits/chosen": -2.358241081237793, + "logits/rejected": -2.3770785331726074, + "logps/chosen": -231.0319061279297, + "logps/rejected": -217.2124481201172, + "loss": 2188.2721, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.0412161760032177, + "rewards/margins": 0.06876528263092041, + "rewards/rejected": -0.10998145490884781, + "step": 1370 + }, + { + "epoch": 0.09, + "learning_rate": 4.5127534336167435e-06, + "logits/chosen": -2.4255921840667725, + "logits/rejected": -2.143770933151245, + "logps/chosen": -252.64108276367188, + "logps/rejected": -215.73745727539062, + "loss": 2182.915, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.05467110127210617, + "rewards/margins": 0.06756436824798584, + "rewards/rejected": -0.12223546206951141, + "step": 1380 + }, + { + "epoch": 0.09, + "learning_rate": 4.5454545454545455e-06, + "logits/chosen": -2.3622450828552246, + "logits/rejected": -2.3135733604431152, + "logps/chosen": -173.19479370117188, + "logps/rejected": -171.76528930664062, + "loss": 2035.6379, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.037360869348049164, + "rewards/margins": 0.08779537677764893, + "rewards/rejected": -0.1251562535762787, + "step": 1390 + }, + { + "epoch": 0.09, + "learning_rate": 4.578155657292348e-06, + "logits/chosen": -2.3347394466400146, + "logits/rejected": -2.300281524658203, + "logps/chosen": -255.4662322998047, + "logps/rejected": -282.37481689453125, + "loss": 1628.1867, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0006058543804101646, + "rewards/margins": 0.17986665666103363, + "rewards/rejected": -0.1804725080728531, + "step": 1400 + }, + { + "epoch": 0.09, + "eval_logits/chosen": -2.3342952728271484, + "eval_logits/rejected": -2.1490938663482666, + "eval_logps/chosen": -234.87429809570312, + "eval_logps/rejected": -225.39889526367188, + "eval_loss": 1933.371337890625, + "eval_rewards/accuracies": 0.6424999833106995, + "eval_rewards/chosen": -0.0286934282630682, + "eval_rewards/margins": 0.1091766282916069, + "eval_rewards/rejected": -0.13787005841732025, + "eval_runtime": 726.3679, + "eval_samples_per_second": 2.753, + "eval_steps_per_second": 1.377, + "step": 1400 + }, + { + "epoch": 0.09, + "learning_rate": 4.610856769130151e-06, + "logits/chosen": -2.3903706073760986, + "logits/rejected": -2.2447268962860107, + "logps/chosen": -240.896240234375, + "logps/rejected": -226.40408325195312, + "loss": 1653.2947, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.017519865185022354, + "rewards/margins": 0.15009410679340363, + "rewards/rejected": -0.16761396825313568, + "step": 1410 + }, + { + "epoch": 0.09, + "learning_rate": 4.643557880967953e-06, + "logits/chosen": -2.3760390281677246, + "logits/rejected": -2.1798462867736816, + "logps/chosen": -205.38589477539062, + "logps/rejected": -211.92306518554688, + "loss": 2097.3713, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.06587065756320953, + "rewards/margins": 0.09036964178085327, + "rewards/rejected": -0.156240314245224, + "step": 1420 + }, + { + "epoch": 0.09, + "learning_rate": 4.676258992805755e-06, + "logits/chosen": -2.3314037322998047, + "logits/rejected": -2.0588929653167725, + "logps/chosen": -278.0765380859375, + "logps/rejected": -250.62222290039062, + "loss": 1547.0062, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.08456582576036453, + "rewards/margins": 0.1469404101371765, + "rewards/rejected": -0.23150622844696045, + "step": 1430 + }, + { + "epoch": 0.09, + "learning_rate": 4.708960104643558e-06, + "logits/chosen": -2.3256824016571045, + "logits/rejected": -2.2896199226379395, + "logps/chosen": -311.00970458984375, + "logps/rejected": -285.14251708984375, + "loss": 2028.9057, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.04799982160329819, + "rewards/margins": 0.10892969369888306, + "rewards/rejected": -0.15692950785160065, + "step": 1440 + }, + { + "epoch": 0.09, + "learning_rate": 4.741661216481361e-06, + "logits/chosen": -2.2723355293273926, + "logits/rejected": -2.2319412231445312, + "logps/chosen": -241.2588653564453, + "logps/rejected": -266.7528076171875, + "loss": 1714.3082, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.049133121967315674, + "rewards/margins": 0.142476424574852, + "rewards/rejected": -0.19160954654216766, + "step": 1450 + }, + { + "epoch": 0.1, + "learning_rate": 4.774362328319163e-06, + "logits/chosen": -2.259704113006592, + "logits/rejected": -2.1161341667175293, + "logps/chosen": -218.36032104492188, + "logps/rejected": -194.8245849609375, + "loss": 2187.3797, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.0257023423910141, + "rewards/margins": 0.06413718312978745, + "rewards/rejected": -0.08983953297138214, + "step": 1460 + }, + { + "epoch": 0.1, + "learning_rate": 4.807063440156966e-06, + "logits/chosen": -2.2820794582366943, + "logits/rejected": -2.0573856830596924, + "logps/chosen": -260.0492858886719, + "logps/rejected": -215.5932159423828, + "loss": 1615.244, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.03260207176208496, + "rewards/margins": 0.1691434383392334, + "rewards/rejected": -0.13654136657714844, + "step": 1470 + }, + { + "epoch": 0.1, + "learning_rate": 4.839764551994769e-06, + "logits/chosen": -2.353740692138672, + "logits/rejected": -2.0564048290252686, + "logps/chosen": -251.4329376220703, + "logps/rejected": -227.07162475585938, + "loss": 2050.0922, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.027244482189416885, + "rewards/margins": 0.13372252881526947, + "rewards/rejected": -0.16096700727939606, + "step": 1480 + }, + { + "epoch": 0.1, + "learning_rate": 4.872465663832571e-06, + "logits/chosen": -2.231527805328369, + "logits/rejected": -2.1027674674987793, + "logps/chosen": -209.59500122070312, + "logps/rejected": -200.1400909423828, + "loss": 1905.0725, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.018365005031228065, + "rewards/margins": 0.11005227267742157, + "rewards/rejected": -0.12841728329658508, + "step": 1490 + }, + { + "epoch": 0.1, + "learning_rate": 4.905166775670373e-06, + "logits/chosen": -2.3522300720214844, + "logits/rejected": -2.024580478668213, + "logps/chosen": -217.8345184326172, + "logps/rejected": -205.73214721679688, + "loss": 1639.7521, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.03925449773669243, + "rewards/margins": 0.15291200578212738, + "rewards/rejected": -0.11365751922130585, + "step": 1500 + }, + { + "epoch": 0.1, + "eval_logits/chosen": -2.3127949237823486, + "eval_logits/rejected": -2.1304714679718018, + "eval_logps/chosen": -231.580810546875, + "eval_logps/rejected": -222.56143188476562, + "eval_loss": 1932.8865966796875, + "eval_rewards/accuracies": 0.6315000057220459, + "eval_rewards/chosen": 0.004241608083248138, + "eval_rewards/margins": 0.11373703181743622, + "eval_rewards/rejected": -0.10949542373418808, + "eval_runtime": 721.6104, + "eval_samples_per_second": 2.772, + "eval_steps_per_second": 1.386, + "step": 1500 + }, + { + "epoch": 0.1, + "learning_rate": 4.9378678875081756e-06, + "logits/chosen": -2.4497547149658203, + "logits/rejected": -2.1537137031555176, + "logps/chosen": -213.77719116210938, + "logps/rejected": -193.41302490234375, + "loss": 1960.867, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.009664192795753479, + "rewards/margins": 0.1064397469162941, + "rewards/rejected": -0.09677554666996002, + "step": 1510 + }, + { + "epoch": 0.1, + "learning_rate": 4.9705689993459784e-06, + "logits/chosen": -2.4343857765197754, + "logits/rejected": -2.002175807952881, + "logps/chosen": -208.59097290039062, + "logps/rejected": -161.232177734375, + "loss": 2100.7449, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.036656323820352554, + "rewards/margins": 0.10354284942150116, + "rewards/rejected": -0.14019916951656342, + "step": 1520 + }, + { + "epoch": 0.1, + "learning_rate": 4.999999934793849e-06, + "logits/chosen": -2.3356330394744873, + "logits/rejected": -2.236433267593384, + "logps/chosen": -248.06582641601562, + "logps/rejected": -229.6756591796875, + "loss": 2047.9803, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.0011256359284743667, + "rewards/margins": 0.11026228964328766, + "rewards/rejected": -0.11138790845870972, + "step": 1530 + }, + { + "epoch": 0.1, + "learning_rate": 4.999992110059814e-06, + "logits/chosen": -2.315499782562256, + "logits/rejected": -2.304673433303833, + "logps/chosen": -274.79168701171875, + "logps/rejected": -262.83050537109375, + "loss": 2078.9848, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.02895769476890564, + "rewards/margins": 0.08152148872613907, + "rewards/rejected": -0.05256379395723343, + "step": 1540 + }, + { + "epoch": 0.1, + "learning_rate": 4.999971244142299e-06, + "logits/chosen": -2.4524457454681396, + "logits/rejected": -2.1760833263397217, + "logps/chosen": -272.97418212890625, + "logps/rejected": -243.3657989501953, + "loss": 1978.4383, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.004789206199347973, + "rewards/margins": 0.09637876600027084, + "rewards/rejected": -0.0915895476937294, + "step": 1550 + }, + { + "epoch": 0.1, + "learning_rate": 4.999937337150149e-06, + "logits/chosen": -2.1443240642547607, + "logits/rejected": -2.1270015239715576, + "logps/chosen": -235.2775421142578, + "logps/rejected": -230.3560333251953, + "loss": 2022.7053, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.0076267956756055355, + "rewards/margins": 0.10459224879741669, + "rewards/rejected": -0.11221903562545776, + "step": 1560 + }, + { + "epoch": 0.1, + "learning_rate": 4.99989038926024e-06, + "logits/chosen": -2.1190567016601562, + "logits/rejected": -2.264734983444214, + "logps/chosen": -203.9278106689453, + "logps/rejected": -216.89608764648438, + "loss": 1955.8473, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.05499737709760666, + "rewards/margins": 0.10766267776489258, + "rewards/rejected": -0.16266006231307983, + "step": 1570 + }, + { + "epoch": 0.1, + "learning_rate": 4.999830400717476e-06, + "logits/chosen": -2.313354015350342, + "logits/rejected": -2.1768441200256348, + "logps/chosen": -292.4227600097656, + "logps/rejected": -290.5616149902344, + "loss": 1720.1881, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.050356674939394, + "rewards/margins": 0.1441434621810913, + "rewards/rejected": -0.1945001184940338, + "step": 1580 + }, + { + "epoch": 0.1, + "learning_rate": 4.999757371834787e-06, + "logits/chosen": -2.1764461994171143, + "logits/rejected": -2.073521614074707, + "logps/chosen": -246.6184539794922, + "logps/rejected": -245.967041015625, + "loss": 1861.9547, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.06799546629190445, + "rewards/margins": 0.14864091575145721, + "rewards/rejected": -0.21663637459278107, + "step": 1590 + }, + { + "epoch": 0.1, + "learning_rate": 4.999671302993125e-06, + "logits/chosen": -2.1240925788879395, + "logits/rejected": -2.0615220069885254, + "logps/chosen": -258.4682922363281, + "logps/rejected": -285.40802001953125, + "loss": 2100.5828, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09887874871492386, + "rewards/margins": 0.11551608145236969, + "rewards/rejected": -0.21439483761787415, + "step": 1600 + }, + { + "epoch": 0.1, + "eval_logits/chosen": -2.305830240249634, + "eval_logits/rejected": -2.1245832443237305, + "eval_logps/chosen": -235.1657257080078, + "eval_logps/rejected": -226.7130126953125, + "eval_loss": 1981.93994140625, + "eval_rewards/accuracies": 0.6244999766349792, + "eval_rewards/chosen": -0.031607791781425476, + "eval_rewards/margins": 0.11940329521894455, + "eval_rewards/rejected": -0.15101107954978943, + "eval_runtime": 723.4625, + "eval_samples_per_second": 2.764, + "eval_steps_per_second": 1.382, + "step": 1600 + }, + { + "epoch": 0.11, + "learning_rate": 4.999572194641471e-06, + "logits/chosen": -2.257770538330078, + "logits/rejected": -2.1237246990203857, + "logps/chosen": -278.04107666015625, + "logps/rejected": -242.87435913085938, + "loss": 1498.3113, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.025227898731827736, + "rewards/margins": 0.17303243279457092, + "rewards/rejected": -0.1982603222131729, + "step": 1610 + }, + { + "epoch": 0.11, + "learning_rate": 4.999460047296819e-06, + "logits/chosen": -2.2606070041656494, + "logits/rejected": -2.1405534744262695, + "logps/chosen": -221.9639129638672, + "logps/rejected": -213.7935333251953, + "loss": 1835.6604, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0635976493358612, + "rewards/margins": 0.12738750874996185, + "rewards/rejected": -0.19098515808582306, + "step": 1620 + }, + { + "epoch": 0.11, + "learning_rate": 4.999334861544186e-06, + "logits/chosen": -2.3754539489746094, + "logits/rejected": -2.0342419147491455, + "logps/chosen": -233.2405242919922, + "logps/rejected": -194.00889587402344, + "loss": 1724.1744, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.03787320479750633, + "rewards/margins": 0.16771630942821503, + "rewards/rejected": -0.20558953285217285, + "step": 1630 + }, + { + "epoch": 0.11, + "learning_rate": 4.999196638036604e-06, + "logits/chosen": -2.4073004722595215, + "logits/rejected": -2.219470500946045, + "logps/chosen": -295.1899719238281, + "logps/rejected": -269.0531921386719, + "loss": 2039.366, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.07774912565946579, + "rewards/margins": 0.09099628031253815, + "rewards/rejected": -0.16874538362026215, + "step": 1640 + }, + { + "epoch": 0.11, + "learning_rate": 4.999045377495111e-06, + "logits/chosen": -2.0742428302764893, + "logits/rejected": -2.323902130126953, + "logps/chosen": -191.41380310058594, + "logps/rejected": -293.4927978515625, + "loss": 1777.3875, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.17079134285449982, + "rewards/margins": 0.14296479523181915, + "rewards/rejected": -0.3137561082839966, + "step": 1650 + }, + { + "epoch": 0.11, + "learning_rate": 4.998881080708759e-06, + "logits/chosen": -2.1812198162078857, + "logits/rejected": -2.102332830429077, + "logps/chosen": -262.5244140625, + "logps/rejected": -238.0368194580078, + "loss": 2214.8494, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.2220289260149002, + "rewards/margins": 0.06457245349884033, + "rewards/rejected": -0.28660139441490173, + "step": 1660 + }, + { + "epoch": 0.11, + "learning_rate": 4.998703748534599e-06, + "logits/chosen": -2.0106849670410156, + "logits/rejected": -1.764390230178833, + "logps/chosen": -263.5819396972656, + "logps/rejected": -224.62124633789062, + "loss": 1848.1158, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2295217514038086, + "rewards/margins": 0.11729858070611954, + "rewards/rejected": -0.34682032465934753, + "step": 1670 + }, + { + "epoch": 0.11, + "learning_rate": 4.998513381897683e-06, + "logits/chosen": -2.2451679706573486, + "logits/rejected": -2.031325340270996, + "logps/chosen": -258.3676452636719, + "logps/rejected": -207.52420043945312, + "loss": 2428.216, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.19123883545398712, + "rewards/margins": 0.06150858476758003, + "rewards/rejected": -0.2527473568916321, + "step": 1680 + }, + { + "epoch": 0.11, + "learning_rate": 4.9983099817910565e-06, + "logits/chosen": -2.2165026664733887, + "logits/rejected": -2.032590389251709, + "logps/chosen": -264.272216796875, + "logps/rejected": -270.38641357421875, + "loss": 2055.751, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.19010308384895325, + "rewards/margins": 0.10592222213745117, + "rewards/rejected": -0.2960253357887268, + "step": 1690 + }, + { + "epoch": 0.11, + "learning_rate": 4.998093549275754e-06, + "logits/chosen": -2.193469285964966, + "logits/rejected": -2.1810290813446045, + "logps/chosen": -273.475830078125, + "logps/rejected": -305.5787048339844, + "loss": 1859.6395, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13141794502735138, + "rewards/margins": 0.10309936851263046, + "rewards/rejected": -0.23451726138591766, + "step": 1700 + }, + { + "epoch": 0.11, + "eval_logits/chosen": -2.2891504764556885, + "eval_logits/rejected": -2.1078803539276123, + "eval_logps/chosen": -243.98208618164062, + "eval_logps/rejected": -234.24464416503906, + "eval_loss": 1915.6973876953125, + "eval_rewards/accuracies": 0.6345000267028809, + "eval_rewards/chosen": -0.11977159231901169, + "eval_rewards/margins": 0.10655605792999268, + "eval_rewards/rejected": -0.22632767260074615, + "eval_runtime": 723.2631, + "eval_samples_per_second": 2.765, + "eval_steps_per_second": 1.383, + "step": 1700 + }, + { + "epoch": 0.11, + "learning_rate": 4.997864085480794e-06, + "logits/chosen": -2.3092598915100098, + "logits/rejected": -2.1490447521209717, + "logps/chosen": -278.25634765625, + "logps/rejected": -275.22210693359375, + "loss": 2087.6984, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11910159885883331, + "rewards/margins": 0.15502703189849854, + "rewards/rejected": -0.27412861585617065, + "step": 1710 + }, + { + "epoch": 0.11, + "learning_rate": 4.997621591603171e-06, + "logits/chosen": -2.287717342376709, + "logits/rejected": -2.093785047531128, + "logps/chosen": -169.4284210205078, + "logps/rejected": -178.6478729248047, + "loss": 1925.2242, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11395607143640518, + "rewards/margins": 0.09904253482818604, + "rewards/rejected": -0.21299858391284943, + "step": 1720 + }, + { + "epoch": 0.11, + "learning_rate": 4.997366068907853e-06, + "logits/chosen": -2.2688260078430176, + "logits/rejected": -2.216841459274292, + "logps/chosen": -261.4203186035156, + "logps/rejected": -248.23538208007812, + "loss": 2182.6373, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.03010564111173153, + "rewards/margins": 0.06690201163291931, + "rewards/rejected": -0.09700765460729599, + "step": 1730 + }, + { + "epoch": 0.11, + "learning_rate": 4.997097518727771e-06, + "logits/chosen": -2.319974422454834, + "logits/rejected": -2.1052706241607666, + "logps/chosen": -226.76400756835938, + "logps/rejected": -204.2520751953125, + "loss": 1985.3566, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.03807578235864639, + "rewards/margins": 0.10610507428646088, + "rewards/rejected": -0.14418084919452667, + "step": 1740 + }, + { + "epoch": 0.11, + "learning_rate": 4.9968159424638155e-06, + "logits/chosen": -2.1506412029266357, + "logits/rejected": -2.3671443462371826, + "logps/chosen": -224.4173126220703, + "logps/rejected": -280.5165710449219, + "loss": 2440.5133, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.06810899078845978, + "rewards/margins": 0.04254139959812164, + "rewards/rejected": -0.11065038293600082, + "step": 1750 + }, + { + "epoch": 0.12, + "learning_rate": 4.9965213415848235e-06, + "logits/chosen": -2.2544147968292236, + "logits/rejected": -1.8740804195404053, + "logps/chosen": -235.32717895507812, + "logps/rejected": -211.6268310546875, + "loss": 1604.1649, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06922627985477448, + "rewards/margins": 0.13159620761871338, + "rewards/rejected": -0.20082247257232666, + "step": 1760 + }, + { + "epoch": 0.12, + "learning_rate": 4.9962137176275805e-06, + "logits/chosen": -2.3310704231262207, + "logits/rejected": -2.1352953910827637, + "logps/chosen": -243.7481231689453, + "logps/rejected": -253.09689331054688, + "loss": 1873.8672, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09452693909406662, + "rewards/margins": 0.09906788170337677, + "rewards/rejected": -0.1935948133468628, + "step": 1770 + }, + { + "epoch": 0.12, + "learning_rate": 4.9958930721968015e-06, + "logits/chosen": -2.2000842094421387, + "logits/rejected": -2.2737386226654053, + "logps/chosen": -235.41317749023438, + "logps/rejected": -254.0403594970703, + "loss": 2234.1324, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.23378005623817444, + "rewards/margins": 0.07125429809093475, + "rewards/rejected": -0.305034339427948, + "step": 1780 + }, + { + "epoch": 0.12, + "learning_rate": 4.995559406965132e-06, + "logits/chosen": -2.4195902347564697, + "logits/rejected": -2.056511402130127, + "logps/chosen": -262.2850646972656, + "logps/rejected": -238.8928680419922, + "loss": 2395.4992, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3036973476409912, + "rewards/margins": 0.06223466247320175, + "rewards/rejected": -0.36593201756477356, + "step": 1790 + }, + { + "epoch": 0.12, + "learning_rate": 4.995212723673131e-06, + "logits/chosen": -2.3763651847839355, + "logits/rejected": -2.149697780609131, + "logps/chosen": -255.087158203125, + "logps/rejected": -214.32907104492188, + "loss": 1918.483, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2488408088684082, + "rewards/margins": 0.09224793314933777, + "rewards/rejected": -0.34108874201774597, + "step": 1800 + }, + { + "epoch": 0.12, + "eval_logits/chosen": -2.3126766681671143, + "eval_logits/rejected": -2.1281871795654297, + "eval_logps/chosen": -253.7191925048828, + "eval_logps/rejected": -242.60064697265625, + "eval_loss": 1937.4407958984375, + "eval_rewards/accuracies": 0.6330000162124634, + "eval_rewards/chosen": -0.21714243292808533, + "eval_rewards/margins": 0.09274499118328094, + "eval_rewards/rejected": -0.30988743901252747, + "eval_runtime": 723.7868, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.382, + "step": 1800 + }, + { + "epoch": 0.12, + "learning_rate": 4.99485302412927e-06, + "logits/chosen": -2.073103189468384, + "logits/rejected": -1.9980010986328125, + "logps/chosen": -227.577880859375, + "logps/rejected": -238.51278686523438, + "loss": 1861.5789, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.19566836953163147, + "rewards/margins": 0.1024223119020462, + "rewards/rejected": -0.2980906367301941, + "step": 1810 + }, + { + "epoch": 0.12, + "learning_rate": 4.994480310209918e-06, + "logits/chosen": -2.2701027393341064, + "logits/rejected": -2.438636541366577, + "logps/chosen": -256.5397033691406, + "logps/rejected": -280.5426330566406, + "loss": 1897.3301, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.14556670188903809, + "rewards/margins": 0.10248645395040512, + "rewards/rejected": -0.2480531483888626, + "step": 1820 + }, + { + "epoch": 0.12, + "learning_rate": 4.994094583859332e-06, + "logits/chosen": -2.3042404651641846, + "logits/rejected": -2.047922372817993, + "logps/chosen": -178.28907775878906, + "logps/rejected": -220.57376098632812, + "loss": 1976.666, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13376356661319733, + "rewards/margins": 0.1025514006614685, + "rewards/rejected": -0.23631496727466583, + "step": 1830 + }, + { + "epoch": 0.12, + "learning_rate": 4.9936958470896525e-06, + "logits/chosen": -2.309569835662842, + "logits/rejected": -2.0754079818725586, + "logps/chosen": -232.97189331054688, + "logps/rejected": -205.6602325439453, + "loss": 1508.1504, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09226727485656738, + "rewards/margins": 0.15591464936733246, + "rewards/rejected": -0.24818190932273865, + "step": 1840 + }, + { + "epoch": 0.12, + "learning_rate": 4.993284101980883e-06, + "logits/chosen": -2.250126361846924, + "logits/rejected": -2.0782506465911865, + "logps/chosen": -252.9325714111328, + "logps/rejected": -232.00851440429688, + "loss": 1648.8354, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.10437178611755371, + "rewards/margins": 0.18708911538124084, + "rewards/rejected": -0.29146090149879456, + "step": 1850 + }, + { + "epoch": 0.12, + "learning_rate": 4.9928593506808885e-06, + "logits/chosen": -2.382662296295166, + "logits/rejected": -2.1939454078674316, + "logps/chosen": -268.0448913574219, + "logps/rejected": -243.6344757080078, + "loss": 2028.7369, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1123049259185791, + "rewards/margins": 0.1256685107946396, + "rewards/rejected": -0.2379734218120575, + "step": 1860 + }, + { + "epoch": 0.12, + "learning_rate": 4.992421595405381e-06, + "logits/chosen": -2.3101463317871094, + "logits/rejected": -2.073124408721924, + "logps/chosen": -242.1526336669922, + "logps/rejected": -184.484375, + "loss": 2065.8262, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.139602392911911, + "rewards/margins": 0.13178659975528717, + "rewards/rejected": -0.2713889479637146, + "step": 1870 + }, + { + "epoch": 0.12, + "learning_rate": 4.991970838437905e-06, + "logits/chosen": -2.208421468734741, + "logits/rejected": -2.1142683029174805, + "logps/chosen": -252.5650177001953, + "logps/rejected": -292.11737060546875, + "loss": 1759.9475, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.27235469222068787, + "rewards/margins": 0.11447665840387344, + "rewards/rejected": -0.3868313431739807, + "step": 1880 + }, + { + "epoch": 0.12, + "learning_rate": 4.9915070821298294e-06, + "logits/chosen": -2.3133842945098877, + "logits/rejected": -1.9845050573349, + "logps/chosen": -193.9105682373047, + "logps/rejected": -192.04551696777344, + "loss": 2260.5314, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.2694074511528015, + "rewards/margins": 0.07223407179117203, + "rewards/rejected": -0.34164154529571533, + "step": 1890 + }, + { + "epoch": 0.12, + "learning_rate": 4.991030328900336e-06, + "logits/chosen": -2.240283250808716, + "logits/rejected": -1.996715784072876, + "logps/chosen": -303.8148498535156, + "logps/rejected": -253.42446899414062, + "loss": 1457.3569, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1924562156200409, + "rewards/margins": 0.19645829498767853, + "rewards/rejected": -0.3889145255088806, + "step": 1900 + }, + { + "epoch": 0.12, + "eval_logits/chosen": -2.2870430946350098, + "eval_logits/rejected": -2.105412483215332, + "eval_logps/chosen": -249.21340942382812, + "eval_logps/rejected": -242.13661193847656, + "eval_loss": 1911.1392822265625, + "eval_rewards/accuracies": 0.6430000066757202, + "eval_rewards/chosen": -0.17208442091941833, + "eval_rewards/margins": 0.13316264748573303, + "eval_rewards/rejected": -0.30524706840515137, + "eval_runtime": 723.5269, + "eval_samples_per_second": 2.764, + "eval_steps_per_second": 1.382, + "step": 1900 + }, + { + "epoch": 0.12, + "learning_rate": 4.9905405812364014e-06, + "logits/chosen": -2.252140522003174, + "logits/rejected": -2.235658645629883, + "logps/chosen": -216.7350616455078, + "logps/rejected": -233.9397735595703, + "loss": 1774.9133, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14438828825950623, + "rewards/margins": 0.14739413559436798, + "rewards/rejected": -0.2917824387550354, + "step": 1910 + }, + { + "epoch": 0.13, + "learning_rate": 4.990037841692791e-06, + "logits/chosen": -2.2116873264312744, + "logits/rejected": -1.971073865890503, + "logps/chosen": -218.53347778320312, + "logps/rejected": -189.02757263183594, + "loss": 1868.9672, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12492616474628448, + "rewards/margins": 0.13778650760650635, + "rewards/rejected": -0.26271265745162964, + "step": 1920 + }, + { + "epoch": 0.13, + "learning_rate": 4.989522112892039e-06, + "logits/chosen": -2.2761216163635254, + "logits/rejected": -2.237509250640869, + "logps/chosen": -214.1681365966797, + "logps/rejected": -224.54296875, + "loss": 1870.507, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.16538745164871216, + "rewards/margins": 0.11264890432357788, + "rewards/rejected": -0.27803635597229004, + "step": 1930 + }, + { + "epoch": 0.13, + "learning_rate": 4.98899339752444e-06, + "logits/chosen": -2.3265023231506348, + "logits/rejected": -2.0911898612976074, + "logps/chosen": -232.5698699951172, + "logps/rejected": -222.6390838623047, + "loss": 1702.9227, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.029675597324967384, + "rewards/margins": 0.19012561440467834, + "rewards/rejected": -0.21980123221874237, + "step": 1940 + }, + { + "epoch": 0.13, + "learning_rate": 4.988451698348033e-06, + "logits/chosen": -2.2512707710266113, + "logits/rejected": -2.224365711212158, + "logps/chosen": -186.9007110595703, + "logps/rejected": -217.6002960205078, + "loss": 2052.41, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08573143184185028, + "rewards/margins": 0.10430196672677994, + "rewards/rejected": -0.19003340601921082, + "step": 1950 + }, + { + "epoch": 0.13, + "learning_rate": 4.987897018188585e-06, + "logits/chosen": -2.209015130996704, + "logits/rejected": -1.9794836044311523, + "logps/chosen": -234.8341064453125, + "logps/rejected": -187.73577880859375, + "loss": 2258.7855, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.12214773893356323, + "rewards/margins": 0.09577733278274536, + "rewards/rejected": -0.2179250717163086, + "step": 1960 + }, + { + "epoch": 0.13, + "learning_rate": 4.9873293599395814e-06, + "logits/chosen": -2.23533296585083, + "logits/rejected": -2.1085968017578125, + "logps/chosen": -210.78250122070312, + "logps/rejected": -220.7260284423828, + "loss": 1796.2572, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.17350736260414124, + "rewards/margins": 0.13926991820335388, + "rewards/rejected": -0.3127772808074951, + "step": 1970 + }, + { + "epoch": 0.13, + "learning_rate": 4.986748726562203e-06, + "logits/chosen": -2.2655141353607178, + "logits/rejected": -2.1425416469573975, + "logps/chosen": -228.8271942138672, + "logps/rejected": -219.7353973388672, + "loss": 1605.6649, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.12972739338874817, + "rewards/margins": 0.1580057144165039, + "rewards/rejected": -0.2877331078052521, + "step": 1980 + }, + { + "epoch": 0.13, + "learning_rate": 4.98615512108532e-06, + "logits/chosen": -2.3564047813415527, + "logits/rejected": -2.2113535404205322, + "logps/chosen": -228.1331329345703, + "logps/rejected": -240.79299926757812, + "loss": 1882.9188, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.06622384488582611, + "rewards/margins": 0.10876498371362686, + "rewards/rejected": -0.17498883605003357, + "step": 1990 + }, + { + "epoch": 0.13, + "learning_rate": 4.985548546605469e-06, + "logits/chosen": -2.1078901290893555, + "logits/rejected": -2.234835147857666, + "logps/chosen": -227.9741668701172, + "logps/rejected": -251.87881469726562, + "loss": 2512.6883, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.14868374168872833, + "rewards/margins": 0.06264106929302216, + "rewards/rejected": -0.2113248109817505, + "step": 2000 + }, + { + "epoch": 0.13, + "eval_logits/chosen": -2.284355878829956, + "eval_logits/rejected": -2.10317325592041, + "eval_logps/chosen": -243.0243682861328, + "eval_logps/rejected": -236.22425842285156, + "eval_loss": 1904.934814453125, + "eval_rewards/accuracies": 0.6395000219345093, + "eval_rewards/chosen": -0.11019425839185715, + "eval_rewards/margins": 0.13592927157878876, + "eval_rewards/rejected": -0.2461235523223877, + "eval_runtime": 724.8151, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 2000 + }, + { + "epoch": 0.13, + "learning_rate": 4.984929006286838e-06, + "logits/chosen": -2.135776996612549, + "logits/rejected": -2.0879735946655273, + "logps/chosen": -220.8446807861328, + "logps/rejected": -230.2923583984375, + "loss": 2360.9186, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11530672013759613, + "rewards/margins": 0.05991410091519356, + "rewards/rejected": -0.1752208173274994, + "step": 2010 + }, + { + "epoch": 0.13, + "learning_rate": 4.984296503361256e-06, + "logits/chosen": -2.372276782989502, + "logits/rejected": -2.01884126663208, + "logps/chosen": -213.36813354492188, + "logps/rejected": -185.35867309570312, + "loss": 1618.9799, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.11448262631893158, + "rewards/margins": 0.13994377851486206, + "rewards/rejected": -0.25442641973495483, + "step": 2020 + }, + { + "epoch": 0.13, + "learning_rate": 4.9836510411281645e-06, + "logits/chosen": -2.2159922122955322, + "logits/rejected": -2.1070516109466553, + "logps/chosen": -291.0960388183594, + "logps/rejected": -272.9174499511719, + "loss": 1484.0746, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.12564203143119812, + "rewards/margins": 0.21223802864551544, + "rewards/rejected": -0.33788007497787476, + "step": 2030 + }, + { + "epoch": 0.13, + "learning_rate": 4.982992622954613e-06, + "logits/chosen": -2.3539059162139893, + "logits/rejected": -2.06502628326416, + "logps/chosen": -299.5829162597656, + "logps/rejected": -200.73965454101562, + "loss": 1749.1727, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.16054727137088776, + "rewards/margins": 0.15325310826301575, + "rewards/rejected": -0.3138003647327423, + "step": 2040 + }, + { + "epoch": 0.13, + "learning_rate": 4.9823212522752325e-06, + "logits/chosen": -2.445986032485962, + "logits/rejected": -2.1705663204193115, + "logps/chosen": -296.69390869140625, + "logps/rejected": -279.83660888671875, + "loss": 1693.1498, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1888018697500229, + "rewards/margins": 0.17178688943386078, + "rewards/rejected": -0.3605887293815613, + "step": 2050 + }, + { + "epoch": 0.13, + "learning_rate": 4.981636932592222e-06, + "logits/chosen": -2.1977176666259766, + "logits/rejected": -2.0966477394104004, + "logps/chosen": -224.46115112304688, + "logps/rejected": -235.4087677001953, + "loss": 1721.5926, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11548157036304474, + "rewards/margins": 0.1496855914592743, + "rewards/rejected": -0.2651671767234802, + "step": 2060 + }, + { + "epoch": 0.14, + "learning_rate": 4.980939667475328e-06, + "logits/chosen": -2.409071445465088, + "logits/rejected": -2.061501979827881, + "logps/chosen": -287.5926513671875, + "logps/rejected": -238.5335693359375, + "loss": 2094.627, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.134174644947052, + "rewards/margins": 0.11532992124557495, + "rewards/rejected": -0.24950456619262695, + "step": 2070 + }, + { + "epoch": 0.14, + "learning_rate": 4.980229460561826e-06, + "logits/chosen": -2.2790915966033936, + "logits/rejected": -2.2238266468048096, + "logps/chosen": -231.64938354492188, + "logps/rejected": -233.92288208007812, + "loss": 1537.9895, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1356803923845291, + "rewards/margins": 0.1757991760969162, + "rewards/rejected": -0.3114795684814453, + "step": 2080 + }, + { + "epoch": 0.14, + "learning_rate": 4.979506315556503e-06, + "logits/chosen": -2.238194465637207, + "logits/rejected": -1.915668249130249, + "logps/chosen": -303.0437316894531, + "logps/rejected": -267.5552673339844, + "loss": 1857.4201, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.16551579535007477, + "rewards/margins": 0.15499567985534668, + "rewards/rejected": -0.32051146030426025, + "step": 2090 + }, + { + "epoch": 0.14, + "learning_rate": 4.9787702362316395e-06, + "logits/chosen": -2.307528018951416, + "logits/rejected": -2.5269463062286377, + "logps/chosen": -206.5220947265625, + "logps/rejected": -241.37844848632812, + "loss": 2032.659, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.15694910287857056, + "rewards/margins": 0.09342299401760101, + "rewards/rejected": -0.25037211179733276, + "step": 2100 + }, + { + "epoch": 0.14, + "eval_logits/chosen": -2.30545711517334, + "eval_logits/rejected": -2.1217041015625, + "eval_logps/chosen": -243.19927978515625, + "eval_logps/rejected": -235.20457458496094, + "eval_loss": 1915.6287841796875, + "eval_rewards/accuracies": 0.6359999775886536, + "eval_rewards/chosen": -0.11194323003292084, + "eval_rewards/margins": 0.12398341298103333, + "eval_rewards/rejected": -0.23592665791511536, + "eval_runtime": 725.809, + "eval_samples_per_second": 2.756, + "eval_steps_per_second": 1.378, + "step": 2100 + }, + { + "epoch": 0.14, + "learning_rate": 4.9780212264269835e-06, + "logits/chosen": -2.2523937225341797, + "logits/rejected": -2.0116162300109863, + "logps/chosen": -199.10507202148438, + "logps/rejected": -190.06492614746094, + "loss": 2266.8332, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.14285996556282043, + "rewards/margins": 0.08095569163560867, + "rewards/rejected": -0.2238156795501709, + "step": 2110 + }, + { + "epoch": 0.14, + "learning_rate": 4.977259290049739e-06, + "logits/chosen": -2.4865453243255615, + "logits/rejected": -1.9351905584335327, + "logps/chosen": -288.5707092285156, + "logps/rejected": -239.4718017578125, + "loss": 1351.2916, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.01628245785832405, + "rewards/margins": 0.20419542491436005, + "rewards/rejected": -0.2204778641462326, + "step": 2120 + }, + { + "epoch": 0.14, + "learning_rate": 4.976484431074538e-06, + "logits/chosen": -2.2364916801452637, + "logits/rejected": -2.1809253692626953, + "logps/chosen": -194.6975860595703, + "logps/rejected": -187.02737426757812, + "loss": 1983.0709, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.008920894004404545, + "rewards/margins": 0.09932135045528412, + "rewards/rejected": -0.10824225097894669, + "step": 2130 + }, + { + "epoch": 0.14, + "learning_rate": 4.975696653543425e-06, + "logits/chosen": -2.3318240642547607, + "logits/rejected": -2.0875277519226074, + "logps/chosen": -249.6975555419922, + "logps/rejected": -251.0378875732422, + "loss": 1705.1627, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.0004876054881606251, + "rewards/margins": 0.13909529149532318, + "rewards/rejected": -0.13958290219306946, + "step": 2140 + }, + { + "epoch": 0.14, + "learning_rate": 4.974895961565835e-06, + "logits/chosen": -2.257631301879883, + "logits/rejected": -1.919825792312622, + "logps/chosen": -178.15733337402344, + "logps/rejected": -195.17355346679688, + "loss": 2029.2096, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.029063880443572998, + "rewards/margins": 0.11788249015808105, + "rewards/rejected": -0.14694637060165405, + "step": 2150 + }, + { + "epoch": 0.14, + "learning_rate": 4.974082359318566e-06, + "logits/chosen": -2.235304355621338, + "logits/rejected": -2.070316791534424, + "logps/chosen": -253.31259155273438, + "logps/rejected": -223.68795776367188, + "loss": 1586.7559, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.003374456660822034, + "rewards/margins": 0.1585448533296585, + "rewards/rejected": -0.15517041087150574, + "step": 2160 + }, + { + "epoch": 0.14, + "learning_rate": 4.973255851045769e-06, + "logits/chosen": -2.2016162872314453, + "logits/rejected": -2.2180228233337402, + "logps/chosen": -219.61465454101562, + "logps/rejected": -198.3446044921875, + "loss": 1641.7762, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.022564545273780823, + "rewards/margins": 0.1547742635011673, + "rewards/rejected": -0.17733880877494812, + "step": 2170 + }, + { + "epoch": 0.14, + "learning_rate": 4.972416441058915e-06, + "logits/chosen": -2.153087854385376, + "logits/rejected": -2.004279613494873, + "logps/chosen": -238.0662078857422, + "logps/rejected": -230.45004272460938, + "loss": 1542.9841, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.08310385793447495, + "rewards/margins": 0.19025757908821106, + "rewards/rejected": -0.273361474275589, + "step": 2180 + }, + { + "epoch": 0.14, + "learning_rate": 4.971564133736777e-06, + "logits/chosen": -2.1095261573791504, + "logits/rejected": -1.938579797744751, + "logps/chosen": -184.16986083984375, + "logps/rejected": -206.4436798095703, + "loss": 1519.6011, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.06169208139181137, + "rewards/margins": 0.17292717099189758, + "rewards/rejected": -0.23461923003196716, + "step": 2190 + }, + { + "epoch": 0.14, + "learning_rate": 4.970698933525409e-06, + "logits/chosen": -2.3949289321899414, + "logits/rejected": -2.128685712814331, + "logps/chosen": -284.95123291015625, + "logps/rejected": -259.23516845703125, + "loss": 2207.576, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.056042980402708054, + "rewards/margins": 0.1395549774169922, + "rewards/rejected": -0.19559794664382935, + "step": 2200 + }, + { + "epoch": 0.14, + "eval_logits/chosen": -2.284047842025757, + "eval_logits/rejected": -2.1025538444519043, + "eval_logps/chosen": -233.87107849121094, + "eval_logps/rejected": -226.02322387695312, + "eval_loss": 1899.6104736328125, + "eval_rewards/accuracies": 0.6399999856948853, + "eval_rewards/chosen": -0.018661215901374817, + "eval_rewards/margins": 0.12545213103294373, + "eval_rewards/rejected": -0.14411336183547974, + "eval_runtime": 723.3244, + "eval_samples_per_second": 2.765, + "eval_steps_per_second": 1.383, + "step": 2200 + }, + { + "epoch": 0.14, + "learning_rate": 4.969820844938118e-06, + "logits/chosen": -2.3908708095550537, + "logits/rejected": -2.084207057952881, + "logps/chosen": -223.781982421875, + "logps/rejected": -184.0693359375, + "loss": 1800.1578, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.011787386611104012, + "rewards/margins": 0.1419001817703247, + "rewards/rejected": -0.15368756651878357, + "step": 2210 + }, + { + "epoch": 0.15, + "learning_rate": 4.968929872555444e-06, + "logits/chosen": -1.9293677806854248, + "logits/rejected": -2.1174938678741455, + "logps/chosen": -226.0688934326172, + "logps/rejected": -267.40545654296875, + "loss": 2410.917, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08934484422206879, + "rewards/margins": 0.056735485792160034, + "rewards/rejected": -0.14608034491539001, + "step": 2220 + }, + { + "epoch": 0.15, + "learning_rate": 4.968026021025137e-06, + "logits/chosen": -2.3432908058166504, + "logits/rejected": -2.1328892707824707, + "logps/chosen": -205.4949951171875, + "logps/rejected": -180.20901489257812, + "loss": 1757.748, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.03930085152387619, + "rewards/margins": 0.1477939933538437, + "rewards/rejected": -0.18709483742713928, + "step": 2230 + }, + { + "epoch": 0.15, + "learning_rate": 4.967109295062128e-06, + "logits/chosen": -2.198868751525879, + "logits/rejected": -1.9802119731903076, + "logps/chosen": -233.25698852539062, + "logps/rejected": -263.6954650878906, + "loss": 2017.6684, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.08255533128976822, + "rewards/margins": 0.09517533332109451, + "rewards/rejected": -0.17773064970970154, + "step": 2240 + }, + { + "epoch": 0.15, + "learning_rate": 4.966179699448509e-06, + "logits/chosen": -2.1731762886047363, + "logits/rejected": -1.978885293006897, + "logps/chosen": -195.97317504882812, + "logps/rejected": -187.7627716064453, + "loss": 2002.8109, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.05218205973505974, + "rewards/margins": 0.09615719318389893, + "rewards/rejected": -0.14833924174308777, + "step": 2250 + }, + { + "epoch": 0.15, + "learning_rate": 4.965237239033506e-06, + "logits/chosen": -2.32716965675354, + "logits/rejected": -2.152055263519287, + "logps/chosen": -299.1788330078125, + "logps/rejected": -276.8256530761719, + "loss": 1632.5648, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.09520121663808823, + "rewards/margins": 0.22428712248802185, + "rewards/rejected": -0.3194883465766907, + "step": 2260 + }, + { + "epoch": 0.15, + "learning_rate": 4.964281918733453e-06, + "logits/chosen": -2.2710909843444824, + "logits/rejected": -2.0281014442443848, + "logps/chosen": -193.1463623046875, + "logps/rejected": -205.0344696044922, + "loss": 1633.4338, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10031914710998535, + "rewards/margins": 0.1634485125541687, + "rewards/rejected": -0.26376765966415405, + "step": 2270 + }, + { + "epoch": 0.15, + "learning_rate": 4.9633137435317715e-06, + "logits/chosen": -2.289713144302368, + "logits/rejected": -1.6606485843658447, + "logps/chosen": -229.876953125, + "logps/rejected": -179.68936157226562, + "loss": 1529.5553, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06981930136680603, + "rewards/margins": 0.17597553133964539, + "rewards/rejected": -0.24579481780529022, + "step": 2280 + }, + { + "epoch": 0.15, + "learning_rate": 4.9623327184789355e-06, + "logits/chosen": -2.3933303356170654, + "logits/rejected": -2.309675931930542, + "logps/chosen": -221.02023315429688, + "logps/rejected": -223.60763549804688, + "loss": 2147.6383, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.03279787302017212, + "rewards/margins": 0.07413671165704727, + "rewards/rejected": -0.10693458467721939, + "step": 2290 + }, + { + "epoch": 0.15, + "learning_rate": 4.9613388486924525e-06, + "logits/chosen": -1.9889920949935913, + "logits/rejected": -2.12893009185791, + "logps/chosen": -181.59829711914062, + "logps/rejected": -211.2786865234375, + "loss": 1761.0438, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.014642806723713875, + "rewards/margins": 0.12300250679254532, + "rewards/rejected": -0.13764533400535583, + "step": 2300 + }, + { + "epoch": 0.15, + "eval_logits/chosen": -2.2852556705474854, + "eval_logits/rejected": -2.1030173301696777, + "eval_logps/chosen": -232.01632690429688, + "eval_logps/rejected": -223.4245147705078, + "eval_loss": 1910.914794921875, + "eval_rewards/accuracies": 0.6365000009536743, + "eval_rewards/chosen": -0.00011374091991456226, + "eval_rewards/margins": 0.1180122047662735, + "eval_rewards/rejected": -0.11812596023082733, + "eval_runtime": 727.0474, + "eval_samples_per_second": 2.751, + "eval_steps_per_second": 1.375, + "step": 2300 + }, + { + "epoch": 0.15, + "learning_rate": 4.960332139356834e-06, + "logits/chosen": -2.2433876991271973, + "logits/rejected": -2.0597805976867676, + "logps/chosen": -208.4698486328125, + "logps/rejected": -196.3324432373047, + "loss": 1667.8039, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.0034699998795986176, + "rewards/margins": 0.14302471280097961, + "rewards/rejected": -0.1395547240972519, + "step": 2310 + }, + { + "epoch": 0.15, + "learning_rate": 4.95931259572357e-06, + "logits/chosen": -2.329911947250366, + "logits/rejected": -1.9796215295791626, + "logps/chosen": -236.82339477539062, + "logps/rejected": -278.1308288574219, + "loss": 1809.4439, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.025833886116743088, + "rewards/margins": 0.15633848309516907, + "rewards/rejected": -0.18217237293720245, + "step": 2320 + }, + { + "epoch": 0.15, + "learning_rate": 4.9582802231111e-06, + "logits/chosen": -2.1584572792053223, + "logits/rejected": -2.244807720184326, + "logps/chosen": -221.17514038085938, + "logps/rejected": -207.26791381835938, + "loss": 1835.1227, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09301011264324188, + "rewards/margins": 0.12971298396587372, + "rewards/rejected": -0.2227230966091156, + "step": 2330 + }, + { + "epoch": 0.15, + "learning_rate": 4.957235026904782e-06, + "logits/chosen": -2.3219685554504395, + "logits/rejected": -2.0044803619384766, + "logps/chosen": -267.11383056640625, + "logps/rejected": -230.76583862304688, + "loss": 1752.1791, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.049614351242780685, + "rewards/margins": 0.14767198264598846, + "rewards/rejected": -0.19728633761405945, + "step": 2340 + }, + { + "epoch": 0.15, + "learning_rate": 4.956177012556875e-06, + "logits/chosen": -2.400453805923462, + "logits/rejected": -2.1781132221221924, + "logps/chosen": -258.6994934082031, + "logps/rejected": -200.09947204589844, + "loss": 2029.8385, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.14506658911705017, + "rewards/margins": 0.12968036532402039, + "rewards/rejected": -0.27474695444107056, + "step": 2350 + }, + { + "epoch": 0.15, + "learning_rate": 4.9551061855864976e-06, + "logits/chosen": -2.069888114929199, + "logits/rejected": -2.1222586631774902, + "logps/chosen": -208.03359985351562, + "logps/rejected": -222.52658081054688, + "loss": 1863.459, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15205471217632294, + "rewards/margins": 0.10282406955957413, + "rewards/rejected": -0.25487881898880005, + "step": 2360 + }, + { + "epoch": 0.16, + "learning_rate": 4.95402255157961e-06, + "logits/chosen": -2.105889320373535, + "logits/rejected": -2.1698150634765625, + "logps/chosen": -196.15615844726562, + "logps/rejected": -273.117431640625, + "loss": 2058.5367, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11099891364574432, + "rewards/margins": 0.12415103614330292, + "rewards/rejected": -0.23514994978904724, + "step": 2370 + }, + { + "epoch": 0.16, + "learning_rate": 4.952926116188977e-06, + "logits/chosen": -2.383444309234619, + "logits/rejected": -2.316694974899292, + "logps/chosen": -195.67141723632812, + "logps/rejected": -240.1870880126953, + "loss": 2270.6312, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.12294076383113861, + "rewards/margins": 0.053111083805561066, + "rewards/rejected": -0.17605182528495789, + "step": 2380 + }, + { + "epoch": 0.16, + "learning_rate": 4.951816885134143e-06, + "logits/chosen": -2.2206435203552246, + "logits/rejected": -2.2017269134521484, + "logps/chosen": -210.9372100830078, + "logps/rejected": -219.2127227783203, + "loss": 2097.6289, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.09384217113256454, + "rewards/margins": 0.09136297553777695, + "rewards/rejected": -0.1852051317691803, + "step": 2390 + }, + { + "epoch": 0.16, + "learning_rate": 4.950694864201399e-06, + "logits/chosen": -2.2418875694274902, + "logits/rejected": -2.162963390350342, + "logps/chosen": -247.30673217773438, + "logps/rejected": -265.2274475097656, + "loss": 1459.2476, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.0862477570772171, + "rewards/margins": 0.1579146683216095, + "rewards/rejected": -0.2441624402999878, + "step": 2400 + }, + { + "epoch": 0.16, + "eval_logits/chosen": -2.260871171951294, + "eval_logits/rejected": -2.0794146060943604, + "eval_logps/chosen": -238.68450927734375, + "eval_logps/rejected": -231.64358520507812, + "eval_loss": 1888.1434326171875, + "eval_rewards/accuracies": 0.6349999904632568, + "eval_rewards/chosen": -0.06679563969373703, + "eval_rewards/margins": 0.13352102041244507, + "eval_rewards/rejected": -0.2003166675567627, + "eval_runtime": 725.1376, + "eval_samples_per_second": 2.758, + "eval_steps_per_second": 1.379, + "step": 2400 + }, + { + "epoch": 0.16, + "learning_rate": 4.9495600592437575e-06, + "logits/chosen": -2.3008065223693848, + "logits/rejected": -2.1299381256103516, + "logps/chosen": -239.4790496826172, + "logps/rejected": -255.85281372070312, + "loss": 1879.6795, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1258460134267807, + "rewards/margins": 0.1319214254617691, + "rewards/rejected": -0.2577674388885498, + "step": 2410 + }, + { + "epoch": 0.16, + "learning_rate": 4.948412476180917e-06, + "logits/chosen": -2.2109181880950928, + "logits/rejected": -1.9310805797576904, + "logps/chosen": -202.6303253173828, + "logps/rejected": -194.5168914794922, + "loss": 1935.1873, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.18554428219795227, + "rewards/margins": 0.14623305201530457, + "rewards/rejected": -0.3317773640155792, + "step": 2420 + }, + { + "epoch": 0.16, + "learning_rate": 4.947252120999232e-06, + "logits/chosen": -2.23991060256958, + "logits/rejected": -1.9719324111938477, + "logps/chosen": -291.8089904785156, + "logps/rejected": -239.60586547851562, + "loss": 2337.2834, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.24704165756702423, + "rewards/margins": 0.06570660322904587, + "rewards/rejected": -0.3127482533454895, + "step": 2430 + }, + { + "epoch": 0.16, + "learning_rate": 4.946078999751683e-06, + "logits/chosen": -2.156996011734009, + "logits/rejected": -2.0907959938049316, + "logps/chosen": -193.2981719970703, + "logps/rejected": -187.05856323242188, + "loss": 1601.603, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13671033084392548, + "rewards/margins": 0.15316611528396606, + "rewards/rejected": -0.28987646102905273, + "step": 2440 + }, + { + "epoch": 0.16, + "learning_rate": 4.944893118557847e-06, + "logits/chosen": -2.0801761150360107, + "logits/rejected": -2.0442826747894287, + "logps/chosen": -223.3832244873047, + "logps/rejected": -181.52224731445312, + "loss": 1935.543, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1689862757921219, + "rewards/margins": 0.11829449236392975, + "rewards/rejected": -0.28728073835372925, + "step": 2450 + }, + { + "epoch": 0.16, + "learning_rate": 4.943694483603861e-06, + "logits/chosen": -2.3802292346954346, + "logits/rejected": -1.9829076528549194, + "logps/chosen": -230.80758666992188, + "logps/rejected": -196.69564819335938, + "loss": 1594.3304, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.04803575947880745, + "rewards/margins": 0.1686701476573944, + "rewards/rejected": -0.21670588850975037, + "step": 2460 + }, + { + "epoch": 0.16, + "learning_rate": 4.9424831011423914e-06, + "logits/chosen": -2.3456430435180664, + "logits/rejected": -2.257657527923584, + "logps/chosen": -293.9297180175781, + "logps/rejected": -263.3369140625, + "loss": 2375.3443, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.05503131076693535, + "rewards/margins": 0.06421846151351929, + "rewards/rejected": -0.11924977600574493, + "step": 2470 + }, + { + "epoch": 0.16, + "learning_rate": 4.9412589774926015e-06, + "logits/chosen": -2.345982789993286, + "logits/rejected": -2.0571885108947754, + "logps/chosen": -275.6372985839844, + "logps/rejected": -241.6995086669922, + "loss": 1729.7469, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.014475801959633827, + "rewards/margins": 0.16685692965984344, + "rewards/rejected": -0.18133273720741272, + "step": 2480 + }, + { + "epoch": 0.16, + "learning_rate": 4.940022119040121e-06, + "logits/chosen": -2.390468120574951, + "logits/rejected": -2.0979933738708496, + "logps/chosen": -295.2787170410156, + "logps/rejected": -278.6435852050781, + "loss": 1984.5342, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.033627700060606, + "rewards/margins": 0.11183059215545654, + "rewards/rejected": -0.14545831084251404, + "step": 2490 + }, + { + "epoch": 0.16, + "learning_rate": 4.93877253223701e-06, + "logits/chosen": -2.327923536300659, + "logits/rejected": -2.0985212326049805, + "logps/chosen": -303.5312194824219, + "logps/rejected": -283.330810546875, + "loss": 1836.8891, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.11259005963802338, + "rewards/margins": 0.11372293531894684, + "rewards/rejected": -0.22631299495697021, + "step": 2500 + }, + { + "epoch": 0.16, + "eval_logits/chosen": -2.2696266174316406, + "eval_logits/rejected": -2.087106466293335, + "eval_logps/chosen": -244.68203735351562, + "eval_logps/rejected": -237.92999267578125, + "eval_loss": 1875.603759765625, + "eval_rewards/accuracies": 0.637499988079071, + "eval_rewards/chosen": -0.12677064538002014, + "eval_rewards/margins": 0.13641037046909332, + "eval_rewards/rejected": -0.26318103075027466, + "eval_runtime": 722.4324, + "eval_samples_per_second": 2.768, + "eval_steps_per_second": 1.384, + "step": 2500 + }, + { + "epoch": 0.16, + "learning_rate": 4.937510223601725e-06, + "logits/chosen": -2.4875149726867676, + "logits/rejected": -2.3256750106811523, + "logps/chosen": -269.61895751953125, + "logps/rejected": -239.8365020751953, + "loss": 1762.4992, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07093014568090439, + "rewards/margins": 0.12428943067789078, + "rewards/rejected": -0.19521956145763397, + "step": 2510 + }, + { + "epoch": 0.16, + "learning_rate": 4.936235199719085e-06, + "logits/chosen": -2.302119016647339, + "logits/rejected": -2.1802175045013428, + "logps/chosen": -183.2852020263672, + "logps/rejected": -172.84878540039062, + "loss": 1602.4494, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.10983192920684814, + "rewards/margins": 0.19742779433727264, + "rewards/rejected": -0.3072597086429596, + "step": 2520 + }, + { + "epoch": 0.17, + "learning_rate": 4.93494746724024e-06, + "logits/chosen": -2.2991228103637695, + "logits/rejected": -2.040062427520752, + "logps/chosen": -238.6068878173828, + "logps/rejected": -275.1326904296875, + "loss": 1426.2948, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.14802417159080505, + "rewards/margins": 0.19065825641155243, + "rewards/rejected": -0.3386824429035187, + "step": 2530 + }, + { + "epoch": 0.17, + "learning_rate": 4.933647032882635e-06, + "logits/chosen": -2.440565586090088, + "logits/rejected": -2.1358885765075684, + "logps/chosen": -258.40179443359375, + "logps/rejected": -229.6172332763672, + "loss": 1756.3088, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1353994458913803, + "rewards/margins": 0.15634958446025848, + "rewards/rejected": -0.2917490005493164, + "step": 2540 + }, + { + "epoch": 0.17, + "learning_rate": 4.932333903429969e-06, + "logits/chosen": -2.1265618801116943, + "logits/rejected": -1.9849107265472412, + "logps/chosen": -208.16122436523438, + "logps/rejected": -188.74383544921875, + "loss": 2189.473, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.15078715980052948, + "rewards/margins": 0.06388656795024872, + "rewards/rejected": -0.2146737277507782, + "step": 2550 + }, + { + "epoch": 0.17, + "learning_rate": 4.931008085732172e-06, + "logits/chosen": -2.3160247802734375, + "logits/rejected": -1.9144550561904907, + "logps/chosen": -227.34365844726562, + "logps/rejected": -189.74508666992188, + "loss": 1769.6924, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.245207279920578, + "rewards/margins": 0.12402795255184174, + "rewards/rejected": -0.36923524737358093, + "step": 2560 + }, + { + "epoch": 0.17, + "learning_rate": 4.9296695867053565e-06, + "logits/chosen": -2.2828354835510254, + "logits/rejected": -2.066901683807373, + "logps/chosen": -319.77569580078125, + "logps/rejected": -278.95794677734375, + "loss": 1323.6122, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.19174179434776306, + "rewards/margins": 0.2514215111732483, + "rewards/rejected": -0.44316330552101135, + "step": 2570 + }, + { + "epoch": 0.17, + "learning_rate": 4.928318413331791e-06, + "logits/chosen": -2.3800201416015625, + "logits/rejected": -2.16902494430542, + "logps/chosen": -237.91647338867188, + "logps/rejected": -230.12765502929688, + "loss": 2151.4758, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2668939232826233, + "rewards/margins": 0.08196422457695007, + "rewards/rejected": -0.34885817766189575, + "step": 2580 + }, + { + "epoch": 0.17, + "learning_rate": 4.926954572659855e-06, + "logits/chosen": -2.1593034267425537, + "logits/rejected": -2.1724724769592285, + "logps/chosen": -267.30853271484375, + "logps/rejected": -300.624267578125, + "loss": 1644.4615, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.25112849473953247, + "rewards/margins": 0.16129925847053528, + "rewards/rejected": -0.41242775321006775, + "step": 2590 + }, + { + "epoch": 0.17, + "learning_rate": 4.925578071804013e-06, + "logits/chosen": -2.196808338165283, + "logits/rejected": -2.143341064453125, + "logps/chosen": -252.46435546875, + "logps/rejected": -320.8917541503906, + "loss": 1885.9926, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.22758595645427704, + "rewards/margins": 0.1101551502943039, + "rewards/rejected": -0.33774110674858093, + "step": 2600 + }, + { + "epoch": 0.17, + "eval_logits/chosen": -2.295515775680542, + "eval_logits/rejected": -2.1104376316070557, + "eval_logps/chosen": -250.35304260253906, + "eval_logps/rejected": -244.417236328125, + "eval_loss": 1840.02197265625, + "eval_rewards/accuracies": 0.6570000052452087, + "eval_rewards/chosen": -0.18348100781440735, + "eval_rewards/margins": 0.14457230269908905, + "eval_rewards/rejected": -0.3280532956123352, + "eval_runtime": 725.3952, + "eval_samples_per_second": 2.757, + "eval_steps_per_second": 1.379, + "step": 2600 + }, + { + "epoch": 0.17, + "learning_rate": 4.924188917944763e-06, + "logits/chosen": -2.357513904571533, + "logits/rejected": -2.1812283992767334, + "logps/chosen": -235.4732208251953, + "logps/rejected": -219.39749145507812, + "loss": 1659.5766, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15468116104602814, + "rewards/margins": 0.17444713413715363, + "rewards/rejected": -0.32912832498550415, + "step": 2610 + }, + { + "epoch": 0.17, + "learning_rate": 4.922787118328617e-06, + "logits/chosen": -2.3777213096618652, + "logits/rejected": -2.0662240982055664, + "logps/chosen": -251.46151733398438, + "logps/rejected": -181.6281280517578, + "loss": 2529.3734, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": -0.21443858742713928, + "rewards/margins": 0.0647946149110794, + "rewards/rejected": -0.2792331874370575, + "step": 2620 + }, + { + "epoch": 0.17, + "learning_rate": 4.921372680268045e-06, + "logits/chosen": -2.359708309173584, + "logits/rejected": -2.0272839069366455, + "logps/chosen": -252.25289916992188, + "logps/rejected": -227.28915405273438, + "loss": 2616.8451, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.2224821299314499, + "rewards/margins": 0.054050058126449585, + "rewards/rejected": -0.2765321731567383, + "step": 2630 + }, + { + "epoch": 0.17, + "learning_rate": 4.919945611141451e-06, + "logits/chosen": -2.4145865440368652, + "logits/rejected": -2.044100284576416, + "logps/chosen": -236.93545532226562, + "logps/rejected": -190.10006713867188, + "loss": 1913.1398, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11700806766748428, + "rewards/margins": 0.11683978140354156, + "rewards/rejected": -0.23384782671928406, + "step": 2640 + }, + { + "epoch": 0.17, + "learning_rate": 4.918505918393125e-06, + "logits/chosen": -2.2917346954345703, + "logits/rejected": -2.1400973796844482, + "logps/chosen": -179.37405395507812, + "logps/rejected": -213.5010223388672, + "loss": 1871.7523, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10343358665704727, + "rewards/margins": 0.10753600299358368, + "rewards/rejected": -0.21096959710121155, + "step": 2650 + }, + { + "epoch": 0.17, + "learning_rate": 4.91705360953321e-06, + "logits/chosen": -2.344794511795044, + "logits/rejected": -2.1143381595611572, + "logps/chosen": -260.46099853515625, + "logps/rejected": -241.50204467773438, + "loss": 1606.4104, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14973367750644684, + "rewards/margins": 0.13696415722370148, + "rewards/rejected": -0.2866978049278259, + "step": 2660 + }, + { + "epoch": 0.17, + "learning_rate": 4.9155886921376615e-06, + "logits/chosen": -2.267577648162842, + "logits/rejected": -2.222317934036255, + "logps/chosen": -222.16390991210938, + "logps/rejected": -253.34878540039062, + "loss": 1950.7385, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.16195516288280487, + "rewards/margins": 0.0951649397611618, + "rewards/rejected": -0.25712013244628906, + "step": 2670 + }, + { + "epoch": 0.18, + "learning_rate": 4.914111173848205e-06, + "logits/chosen": -2.316715717315674, + "logits/rejected": -2.2697932720184326, + "logps/chosen": -250.2243194580078, + "logps/rejected": -240.92874145507812, + "loss": 2477.1484, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": -0.17192703485488892, + "rewards/margins": 0.06122405454516411, + "rewards/rejected": -0.2331511229276657, + "step": 2680 + }, + { + "epoch": 0.18, + "learning_rate": 4.9126210623723e-06, + "logits/chosen": -2.0935120582580566, + "logits/rejected": -2.2726893424987793, + "logps/chosen": -212.24560546875, + "logps/rejected": -259.00762939453125, + "loss": 1884.5598, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11872206628322601, + "rewards/margins": 0.1299000084400177, + "rewards/rejected": -0.24862205982208252, + "step": 2690 + }, + { + "epoch": 0.18, + "learning_rate": 4.911118365483098e-06, + "logits/chosen": -2.190582752227783, + "logits/rejected": -2.280961275100708, + "logps/chosen": -216.3970947265625, + "logps/rejected": -242.3699951171875, + "loss": 1692.8492, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1464289426803589, + "rewards/margins": 0.165466770529747, + "rewards/rejected": -0.3118956685066223, + "step": 2700 + }, + { + "epoch": 0.18, + "eval_logits/chosen": -2.3050317764282227, + "eval_logits/rejected": -2.1195755004882812, + "eval_logps/chosen": -247.22756958007812, + "eval_logps/rejected": -238.92596435546875, + "eval_loss": 1861.056640625, + "eval_rewards/accuracies": 0.6549999713897705, + "eval_rewards/chosen": -0.15222610533237457, + "eval_rewards/margins": 0.12091454118490219, + "eval_rewards/rejected": -0.27314069867134094, + "eval_runtime": 724.7776, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 2700 + }, + { + "epoch": 0.18, + "learning_rate": 4.909603091019403e-06, + "logits/chosen": -2.4673802852630615, + "logits/rejected": -2.0942001342773438, + "logps/chosen": -246.7534942626953, + "logps/rejected": -226.46493530273438, + "loss": 1654.3146, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.10376157611608505, + "rewards/margins": 0.16331735253334045, + "rewards/rejected": -0.2670789361000061, + "step": 2710 + }, + { + "epoch": 0.18, + "learning_rate": 4.908075246885626e-06, + "logits/chosen": -2.241844892501831, + "logits/rejected": -2.1684601306915283, + "logps/chosen": -163.9541473388672, + "logps/rejected": -142.56719970703125, + "loss": 2271.8395, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.13226650655269623, + "rewards/margins": 0.06004362553358078, + "rewards/rejected": -0.1923101395368576, + "step": 2720 + }, + { + "epoch": 0.18, + "learning_rate": 4.906534841051755e-06, + "logits/chosen": -2.1116034984588623, + "logits/rejected": -2.1748390197753906, + "logps/chosen": -256.3478088378906, + "logps/rejected": -277.2479248046875, + "loss": 2043.9254, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12944331765174866, + "rewards/margins": 0.09864501655101776, + "rewards/rejected": -0.2280883491039276, + "step": 2730 + }, + { + "epoch": 0.18, + "learning_rate": 4.904981881553297e-06, + "logits/chosen": -2.37188982963562, + "logits/rejected": -2.04917311668396, + "logps/chosen": -238.0791015625, + "logps/rejected": -181.63294982910156, + "loss": 2043.5838, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1388280689716339, + "rewards/margins": 0.07672025263309479, + "rewards/rejected": -0.2155483067035675, + "step": 2740 + }, + { + "epoch": 0.18, + "learning_rate": 4.903416376491252e-06, + "logits/chosen": -2.3755104541778564, + "logits/rejected": -1.9843056201934814, + "logps/chosen": -289.67669677734375, + "logps/rejected": -264.0829772949219, + "loss": 1753.6057, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11384458839893341, + "rewards/margins": 0.14998391270637512, + "rewards/rejected": -0.26382848620414734, + "step": 2750 + }, + { + "epoch": 0.18, + "learning_rate": 4.90183833403206e-06, + "logits/chosen": -2.438305377960205, + "logits/rejected": -2.282226800918579, + "logps/chosen": -271.61114501953125, + "logps/rejected": -249.8431396484375, + "loss": 1752.6217, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.07398039102554321, + "rewards/margins": 0.14484994113445282, + "rewards/rejected": -0.21883034706115723, + "step": 2760 + }, + { + "epoch": 0.18, + "learning_rate": 4.900247762407564e-06, + "logits/chosen": -2.2102179527282715, + "logits/rejected": -1.9937061071395874, + "logps/chosen": -181.43264770507812, + "logps/rejected": -217.74343872070312, + "loss": 1837.5686, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.09140662103891373, + "rewards/margins": 0.14268136024475098, + "rewards/rejected": -0.2340880185365677, + "step": 2770 + }, + { + "epoch": 0.18, + "learning_rate": 4.898644669914965e-06, + "logits/chosen": -2.2669076919555664, + "logits/rejected": -2.18135929107666, + "logps/chosen": -231.2422332763672, + "logps/rejected": -231.5166015625, + "loss": 1757.7465, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07169830054044724, + "rewards/margins": 0.14100399613380432, + "rewards/rejected": -0.21270231902599335, + "step": 2780 + }, + { + "epoch": 0.18, + "learning_rate": 4.897029064916778e-06, + "logits/chosen": -2.0898585319519043, + "logits/rejected": -1.890570044517517, + "logps/chosen": -217.30859375, + "logps/rejected": -213.62460327148438, + "loss": 2316.4094, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.09158669412136078, + "rewards/margins": 0.08704870194196701, + "rewards/rejected": -0.1786354035139084, + "step": 2790 + }, + { + "epoch": 0.18, + "learning_rate": 4.895400955840791e-06, + "logits/chosen": -2.440882444381714, + "logits/rejected": -1.7953065633773804, + "logps/chosen": -222.1812286376953, + "logps/rejected": -193.8184356689453, + "loss": 1884.091, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.01829930767416954, + "rewards/margins": 0.13334982097148895, + "rewards/rejected": -0.15164914727210999, + "step": 2800 + }, + { + "epoch": 0.18, + "eval_logits/chosen": -2.2834653854370117, + "eval_logits/rejected": -2.0998635292053223, + "eval_logps/chosen": -236.10433959960938, + "eval_logps/rejected": -228.92465209960938, + "eval_loss": 1863.062255859375, + "eval_rewards/accuracies": 0.6439999938011169, + "eval_rewards/chosen": -0.04099392145872116, + "eval_rewards/margins": 0.13213366270065308, + "eval_rewards/rejected": -0.17312759160995483, + "eval_runtime": 727.0631, + "eval_samples_per_second": 2.751, + "eval_steps_per_second": 1.375, + "step": 2800 + }, + { + "epoch": 0.18, + "learning_rate": 4.893760351180018e-06, + "logits/chosen": -2.280066967010498, + "logits/rejected": -2.216137170791626, + "logps/chosen": -204.33236694335938, + "logps/rejected": -221.7998504638672, + "loss": 1961.8484, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.05791081115603447, + "rewards/margins": 0.12366513907909393, + "rewards/rejected": -0.1815759390592575, + "step": 2810 + }, + { + "epoch": 0.18, + "learning_rate": 4.892107259492657e-06, + "logits/chosen": -2.232470750808716, + "logits/rejected": -1.9912874698638916, + "logps/chosen": -243.64651489257812, + "logps/rejected": -255.9471893310547, + "loss": 2238.5156, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.03787205368280411, + "rewards/margins": 0.06281791627407074, + "rewards/rejected": -0.10068996995687485, + "step": 2820 + }, + { + "epoch": 0.19, + "learning_rate": 4.890441689402042e-06, + "logits/chosen": -2.360664129257202, + "logits/rejected": -2.188075542449951, + "logps/chosen": -341.1266174316406, + "logps/rejected": -309.93060302734375, + "loss": 1642.7727, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.05621671676635742, + "rewards/margins": 0.168039932847023, + "rewards/rejected": -0.22425666451454163, + "step": 2830 + }, + { + "epoch": 0.19, + "learning_rate": 4.888763649596606e-06, + "logits/chosen": -2.3540730476379395, + "logits/rejected": -2.131410598754883, + "logps/chosen": -217.951171875, + "logps/rejected": -220.7685089111328, + "loss": 2136.8172, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07789482176303864, + "rewards/margins": 0.11618749797344208, + "rewards/rejected": -0.1940823197364807, + "step": 2840 + }, + { + "epoch": 0.19, + "learning_rate": 4.887073148829824e-06, + "logits/chosen": -2.2597062587738037, + "logits/rejected": -2.1080522537231445, + "logps/chosen": -274.22418212890625, + "logps/rejected": -267.17230224609375, + "loss": 1813.1502, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.07320614904165268, + "rewards/margins": 0.15883690118789673, + "rewards/rejected": -0.23204305768013, + "step": 2850 + }, + { + "epoch": 0.19, + "learning_rate": 4.885370195920177e-06, + "logits/chosen": -2.1275761127471924, + "logits/rejected": -2.072493314743042, + "logps/chosen": -201.2732391357422, + "logps/rejected": -205.1205291748047, + "loss": 1935.8369, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1594954878091812, + "rewards/margins": 0.12420445680618286, + "rewards/rejected": -0.2836999297142029, + "step": 2860 + }, + { + "epoch": 0.19, + "learning_rate": 4.883654799751101e-06, + "logits/chosen": -2.0364058017730713, + "logits/rejected": -2.2626266479492188, + "logps/chosen": -233.87698364257812, + "logps/rejected": -280.73248291015625, + "loss": 1835.7615, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1816781461238861, + "rewards/margins": 0.12115390598773956, + "rewards/rejected": -0.3028320372104645, + "step": 2870 + }, + { + "epoch": 0.19, + "learning_rate": 4.8819269692709435e-06, + "logits/chosen": -2.3218634128570557, + "logits/rejected": -2.165917158126831, + "logps/chosen": -288.1819152832031, + "logps/rejected": -229.6846160888672, + "loss": 1725.4633, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.21135666966438293, + "rewards/margins": 0.13864858448505402, + "rewards/rejected": -0.35000520944595337, + "step": 2880 + }, + { + "epoch": 0.19, + "learning_rate": 4.880186713492915e-06, + "logits/chosen": -2.1620290279388428, + "logits/rejected": -1.9891254901885986, + "logps/chosen": -253.9564971923828, + "logps/rejected": -210.16915893554688, + "loss": 2131.4555, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.26239141821861267, + "rewards/margins": 0.09834585338830948, + "rewards/rejected": -0.36073732376098633, + "step": 2890 + }, + { + "epoch": 0.19, + "learning_rate": 4.878434041495041e-06, + "logits/chosen": -2.218656063079834, + "logits/rejected": -2.3371100425720215, + "logps/chosen": -256.22857666015625, + "logps/rejected": -279.4388427734375, + "loss": 1507.8515, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.19062820076942444, + "rewards/margins": 0.21348030865192413, + "rewards/rejected": -0.40410852432250977, + "step": 2900 + }, + { + "epoch": 0.19, + "eval_logits/chosen": -2.2201428413391113, + "eval_logits/rejected": -2.0429089069366455, + "eval_logps/chosen": -251.0389404296875, + "eval_logps/rejected": -243.269775390625, + "eval_loss": 1893.5802001953125, + "eval_rewards/accuracies": 0.6434999704360962, + "eval_rewards/chosen": -0.19034002721309662, + "eval_rewards/margins": 0.12623876333236694, + "eval_rewards/rejected": -0.31657880544662476, + "eval_runtime": 724.4377, + "eval_samples_per_second": 2.761, + "eval_steps_per_second": 1.38, + "step": 2900 + }, + { + "epoch": 0.19, + "learning_rate": 4.876668962420117e-06, + "logits/chosen": -2.2052314281463623, + "logits/rejected": -1.9575411081314087, + "logps/chosen": -305.30145263671875, + "logps/rejected": -259.78326416015625, + "loss": 1727.6314, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1565035879611969, + "rewards/margins": 0.14097970724105835, + "rewards/rejected": -0.29748329520225525, + "step": 2910 + }, + { + "epoch": 0.19, + "learning_rate": 4.87489148547566e-06, + "logits/chosen": -2.212196111679077, + "logits/rejected": -2.0677270889282227, + "logps/chosen": -280.8736572265625, + "logps/rejected": -257.7037048339844, + "loss": 2261.3482, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.21015194058418274, + "rewards/margins": 0.10002975165843964, + "rewards/rejected": -0.31018170714378357, + "step": 2920 + }, + { + "epoch": 0.19, + "learning_rate": 4.873101619933862e-06, + "logits/chosen": -2.4206960201263428, + "logits/rejected": -2.0775089263916016, + "logps/chosen": -279.95599365234375, + "logps/rejected": -240.8511962890625, + "loss": 2119.6105, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.15428827702999115, + "rewards/margins": 0.10210802406072617, + "rewards/rejected": -0.2563963234424591, + "step": 2930 + }, + { + "epoch": 0.19, + "learning_rate": 4.8712993751315385e-06, + "logits/chosen": -2.2349343299865723, + "logits/rejected": -2.1785027980804443, + "logps/chosen": -132.24380493164062, + "logps/rejected": -144.46116638183594, + "loss": 1970.4197, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.11585874855518341, + "rewards/margins": 0.08315489441156387, + "rewards/rejected": -0.1990136355161667, + "step": 2940 + }, + { + "epoch": 0.19, + "learning_rate": 4.869484760470079e-06, + "logits/chosen": -2.2715892791748047, + "logits/rejected": -2.0801520347595215, + "logps/chosen": -198.9450225830078, + "logps/rejected": -173.6745147705078, + "loss": 1885.7426, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10284195095300674, + "rewards/margins": 0.1102866679430008, + "rewards/rejected": -0.21312859654426575, + "step": 2950 + }, + { + "epoch": 0.19, + "learning_rate": 4.867657785415404e-06, + "logits/chosen": -2.2036194801330566, + "logits/rejected": -1.9305412769317627, + "logps/chosen": -263.3036193847656, + "logps/rejected": -236.1821746826172, + "loss": 1663.068, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1420459896326065, + "rewards/margins": 0.14623163640499115, + "rewards/rejected": -0.28827762603759766, + "step": 2960 + }, + { + "epoch": 0.19, + "learning_rate": 4.865818459497911e-06, + "logits/chosen": -2.438999652862549, + "logits/rejected": -1.9822498559951782, + "logps/chosen": -299.4256591796875, + "logps/rejected": -227.34402465820312, + "loss": 1838.8684, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1524694859981537, + "rewards/margins": 0.12715639173984528, + "rewards/rejected": -0.27962589263916016, + "step": 2970 + }, + { + "epoch": 0.19, + "learning_rate": 4.863966792312423e-06, + "logits/chosen": -2.2882912158966064, + "logits/rejected": -2.063962459564209, + "logps/chosen": -253.9712677001953, + "logps/rejected": -231.25228881835938, + "loss": 1536.9635, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12512370944023132, + "rewards/margins": 0.18332403898239136, + "rewards/rejected": -0.30844777822494507, + "step": 2980 + }, + { + "epoch": 0.2, + "learning_rate": 4.862102793518145e-06, + "logits/chosen": -2.148599147796631, + "logits/rejected": -2.2063260078430176, + "logps/chosen": -214.335693359375, + "logps/rejected": -231.567138671875, + "loss": 2156.0416, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.20053806900978088, + "rewards/margins": 0.10306447744369507, + "rewards/rejected": -0.30360254645347595, + "step": 2990 + }, + { + "epoch": 0.2, + "learning_rate": 4.8602264728386075e-06, + "logits/chosen": -2.243011951446533, + "logits/rejected": -2.100947618484497, + "logps/chosen": -262.6037902832031, + "logps/rejected": -279.33447265625, + "loss": 1346.5843, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10104241222143173, + "rewards/margins": 0.20149335265159607, + "rewards/rejected": -0.3025357723236084, + "step": 3000 + }, + { + "epoch": 0.2, + "eval_logits/chosen": -2.2279601097106934, + "eval_logits/rejected": -2.049701452255249, + "eval_logps/chosen": -240.255126953125, + "eval_logps/rejected": -234.92210388183594, + "eval_loss": 1877.9161376953125, + "eval_rewards/accuracies": 0.6504999995231628, + "eval_rewards/chosen": -0.08250155299901962, + "eval_rewards/margins": 0.15060047805309296, + "eval_rewards/rejected": -0.23310202360153198, + "eval_runtime": 725.2528, + "eval_samples_per_second": 2.758, + "eval_steps_per_second": 1.379, + "step": 3000 + }, + { + "epoch": 0.2, + "learning_rate": 4.858337840061616e-06, + "logits/chosen": -2.2241852283477783, + "logits/rejected": -2.142771005630493, + "logps/chosen": -188.62991333007812, + "logps/rejected": -255.5631561279297, + "loss": 2052.9611, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.05549541115760803, + "rewards/margins": 0.13284072279930115, + "rewards/rejected": -0.18833611905574799, + "step": 3010 + }, + { + "epoch": 0.2, + "learning_rate": 4.856436905039208e-06, + "logits/chosen": -2.2276177406311035, + "logits/rejected": -2.082777738571167, + "logps/chosen": -218.30136108398438, + "logps/rejected": -198.83096313476562, + "loss": 1747.2271, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07096312940120697, + "rewards/margins": 0.14916923642158508, + "rewards/rejected": -0.22013239562511444, + "step": 3020 + }, + { + "epoch": 0.2, + "learning_rate": 4.854523677687588e-06, + "logits/chosen": -2.1049065589904785, + "logits/rejected": -2.1721158027648926, + "logps/chosen": -184.58395385742188, + "logps/rejected": -217.59536743164062, + "loss": 1815.2102, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.03338410705327988, + "rewards/margins": 0.14614860713481903, + "rewards/rejected": -0.1795327216386795, + "step": 3030 + }, + { + "epoch": 0.2, + "learning_rate": 4.85259816798709e-06, + "logits/chosen": -2.30084490776062, + "logits/rejected": -1.7834408283233643, + "logps/chosen": -287.6846618652344, + "logps/rejected": -224.79928588867188, + "loss": 1338.8376, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 0.0007740993169136345, + "rewards/margins": 0.17670534551143646, + "rewards/rejected": -0.17593125998973846, + "step": 3040 + }, + { + "epoch": 0.2, + "learning_rate": 4.850660385982114e-06, + "logits/chosen": -2.304020404815674, + "logits/rejected": -2.145203113555908, + "logps/chosen": -250.6403045654297, + "logps/rejected": -208.04324340820312, + "loss": 1921.6279, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.02921169437468052, + "rewards/margins": 0.1268424391746521, + "rewards/rejected": -0.15605410933494568, + "step": 3050 + }, + { + "epoch": 0.2, + "learning_rate": 4.848710341781081e-06, + "logits/chosen": -2.060889720916748, + "logits/rejected": -2.165678024291992, + "logps/chosen": -180.858642578125, + "logps/rejected": -178.9075164794922, + "loss": 2665.2871, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": -0.06957096606492996, + "rewards/margins": 0.037756115198135376, + "rewards/rejected": -0.10732706636190414, + "step": 3060 + }, + { + "epoch": 0.2, + "learning_rate": 4.846748045556377e-06, + "logits/chosen": -2.2592568397521973, + "logits/rejected": -1.953974723815918, + "logps/chosen": -240.12161254882812, + "logps/rejected": -192.3448944091797, + "loss": 1955.2693, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.006345144007354975, + "rewards/margins": 0.1273423135280609, + "rewards/rejected": -0.13368746638298035, + "step": 3070 + }, + { + "epoch": 0.2, + "learning_rate": 4.8447735075442995e-06, + "logits/chosen": -2.1320343017578125, + "logits/rejected": -2.200254201889038, + "logps/chosen": -198.74644470214844, + "logps/rejected": -212.51657104492188, + "loss": 1535.8741, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.011842348612844944, + "rewards/margins": 0.1717064082622528, + "rewards/rejected": -0.18354876339435577, + "step": 3080 + }, + { + "epoch": 0.2, + "learning_rate": 4.8427867380450075e-06, + "logits/chosen": -2.319826602935791, + "logits/rejected": -1.9436454772949219, + "logps/chosen": -234.8804473876953, + "logps/rejected": -204.9860382080078, + "loss": 1767.59, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.04541159048676491, + "rewards/margins": 0.16355356574058533, + "rewards/rejected": -0.20896515250205994, + "step": 3090 + }, + { + "epoch": 0.2, + "learning_rate": 4.840787747422462e-06, + "logits/chosen": -2.2996203899383545, + "logits/rejected": -2.048703193664551, + "logps/chosen": -199.39035034179688, + "logps/rejected": -182.53619384765625, + "loss": 1925.325, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.03734218329191208, + "rewards/margins": 0.13403446972370148, + "rewards/rejected": -0.17137663066387177, + "step": 3100 + }, + { + "epoch": 0.2, + "eval_logits/chosen": -2.252258777618408, + "eval_logits/rejected": -2.0728886127471924, + "eval_logps/chosen": -237.34962463378906, + "eval_logps/rejected": -231.37498474121094, + "eval_loss": 1869.25146484375, + "eval_rewards/accuracies": 0.6434999704360962, + "eval_rewards/chosen": -0.05344657227396965, + "eval_rewards/margins": 0.1441843956708908, + "eval_rewards/rejected": -0.19763098657131195, + "eval_runtime": 724.8757, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 3100 + }, + { + "epoch": 0.2, + "learning_rate": 4.838776546104378e-06, + "logits/chosen": -2.225860118865967, + "logits/rejected": -2.2323122024536133, + "logps/chosen": -283.26953125, + "logps/rejected": -259.8985595703125, + "loss": 1587.6234, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.023162994533777237, + "rewards/margins": 0.17030958831310272, + "rewards/rejected": -0.19347259402275085, + "step": 3110 + }, + { + "epoch": 0.2, + "learning_rate": 4.836753144582168e-06, + "logits/chosen": -2.1938886642456055, + "logits/rejected": -1.9944331645965576, + "logps/chosen": -249.22616577148438, + "logps/rejected": -242.0547332763672, + "loss": 1700.248, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.0699460357427597, + "rewards/margins": 0.16941139101982117, + "rewards/rejected": -0.23935742676258087, + "step": 3120 + }, + { + "epoch": 0.2, + "learning_rate": 4.834717553410884e-06, + "logits/chosen": -2.2574543952941895, + "logits/rejected": -2.0224785804748535, + "logps/chosen": -193.8590850830078, + "logps/rejected": -225.0542449951172, + "loss": 1770.31, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.045830778777599335, + "rewards/margins": 0.17525990307331085, + "rewards/rejected": -0.2210906744003296, + "step": 3130 + }, + { + "epoch": 0.21, + "learning_rate": 4.832669783209167e-06, + "logits/chosen": -2.1316919326782227, + "logits/rejected": -2.170029401779175, + "logps/chosen": -252.727294921875, + "logps/rejected": -261.55657958984375, + "loss": 2120.9629, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08301271498203278, + "rewards/margins": 0.0828041061758995, + "rewards/rejected": -0.16581682860851288, + "step": 3140 + }, + { + "epoch": 0.21, + "learning_rate": 4.8306098446591895e-06, + "logits/chosen": -1.8441492319107056, + "logits/rejected": -2.0120224952697754, + "logps/chosen": -182.72091674804688, + "logps/rejected": -216.39108276367188, + "loss": 2055.383, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.06452050805091858, + "rewards/margins": 0.08738856017589569, + "rewards/rejected": -0.15190906822681427, + "step": 3150 + }, + { + "epoch": 0.21, + "learning_rate": 4.828537748506601e-06, + "logits/chosen": -2.341561794281006, + "logits/rejected": -2.0833611488342285, + "logps/chosen": -279.31024169921875, + "logps/rejected": -233.48934936523438, + "loss": 2085.3748, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.05911997705698013, + "rewards/margins": 0.10425326973199844, + "rewards/rejected": -0.16337324678897858, + "step": 3160 + }, + { + "epoch": 0.21, + "learning_rate": 4.826453505560469e-06, + "logits/chosen": -2.0433287620544434, + "logits/rejected": -2.0206546783447266, + "logps/chosen": -195.5936279296875, + "logps/rejected": -193.37220764160156, + "loss": 2035.9855, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.04614509269595146, + "rewards/margins": 0.11475135385990143, + "rewards/rejected": -0.1608964502811432, + "step": 3170 + }, + { + "epoch": 0.21, + "learning_rate": 4.824357126693226e-06, + "logits/chosen": -2.1624441146850586, + "logits/rejected": -1.7851953506469727, + "logps/chosen": -264.33087158203125, + "logps/rejected": -229.04415893554688, + "loss": 1915.7865, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.05587694048881531, + "rewards/margins": 0.1344069540500641, + "rewards/rejected": -0.1902839094400406, + "step": 3180 + }, + { + "epoch": 0.21, + "learning_rate": 4.8222486228406105e-06, + "logits/chosen": -2.3245861530303955, + "logits/rejected": -2.0476653575897217, + "logps/chosen": -221.2161102294922, + "logps/rejected": -202.4019012451172, + "loss": 1605.2804, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09975741058588028, + "rewards/margins": 0.15866848826408386, + "rewards/rejected": -0.25842589139938354, + "step": 3190 + }, + { + "epoch": 0.21, + "learning_rate": 4.820128005001612e-06, + "logits/chosen": -2.0240657329559326, + "logits/rejected": -1.9451675415039062, + "logps/chosen": -227.9483184814453, + "logps/rejected": -223.92037963867188, + "loss": 1560.9117, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08954762667417526, + "rewards/margins": 0.17514896392822266, + "rewards/rejected": -0.2646965980529785, + "step": 3200 + }, + { + "epoch": 0.21, + "eval_logits/chosen": -2.276754379272461, + "eval_logits/rejected": -2.0942800045013428, + "eval_logps/chosen": -241.58523559570312, + "eval_logps/rejected": -235.07289123535156, + "eval_loss": 1867.82958984375, + "eval_rewards/accuracies": 0.6434999704360962, + "eval_rewards/chosen": -0.09580282866954803, + "eval_rewards/margins": 0.13880720734596252, + "eval_rewards/rejected": -0.23461003601551056, + "eval_runtime": 723.6984, + "eval_samples_per_second": 2.764, + "eval_steps_per_second": 1.382, + "step": 3200 + }, + { + "epoch": 0.21, + "learning_rate": 4.817995284238412e-06, + "logits/chosen": -2.069889783859253, + "logits/rejected": -2.154778480529785, + "logps/chosen": -202.47483825683594, + "logps/rejected": -244.8623046875, + "loss": 1725.6744, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.059504859149456024, + "rewards/margins": 0.14902649819850922, + "rewards/rejected": -0.20853137969970703, + "step": 3210 + }, + { + "epoch": 0.21, + "learning_rate": 4.815850471676327e-06, + "logits/chosen": -2.1980929374694824, + "logits/rejected": -2.0959055423736572, + "logps/chosen": -249.5984649658203, + "logps/rejected": -258.99560546875, + "loss": 1917.734, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11184433847665787, + "rewards/margins": 0.13644441962242126, + "rewards/rejected": -0.24828878045082092, + "step": 3220 + }, + { + "epoch": 0.21, + "learning_rate": 4.813693578503751e-06, + "logits/chosen": -2.2432820796966553, + "logits/rejected": -2.066694736480713, + "logps/chosen": -302.91668701171875, + "logps/rejected": -265.0500793457031, + "loss": 1498.3356, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.06409894675016403, + "rewards/margins": 0.17591038346290588, + "rewards/rejected": -0.2400093525648117, + "step": 3230 + }, + { + "epoch": 0.21, + "learning_rate": 4.811524615972093e-06, + "logits/chosen": -2.2296462059020996, + "logits/rejected": -2.0955986976623535, + "logps/chosen": -240.7917938232422, + "logps/rejected": -265.7709045410156, + "loss": 1962.7566, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10856205224990845, + "rewards/margins": 0.16776728630065918, + "rewards/rejected": -0.2763293385505676, + "step": 3240 + }, + { + "epoch": 0.21, + "learning_rate": 4.809343595395724e-06, + "logits/chosen": -2.424299716949463, + "logits/rejected": -2.241176128387451, + "logps/chosen": -200.36248779296875, + "logps/rejected": -184.1348419189453, + "loss": 1954.6293, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1130683422088623, + "rewards/margins": 0.1436108499765396, + "rewards/rejected": -0.2566792070865631, + "step": 3250 + }, + { + "epoch": 0.21, + "learning_rate": 4.807150528151918e-06, + "logits/chosen": -2.257136344909668, + "logits/rejected": -2.0939416885375977, + "logps/chosen": -171.06979370117188, + "logps/rejected": -207.30239868164062, + "loss": 1552.4317, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.04377254471182823, + "rewards/margins": 0.18365688621997833, + "rewards/rejected": -0.22742946445941925, + "step": 3260 + }, + { + "epoch": 0.21, + "learning_rate": 4.804945425680787e-06, + "logits/chosen": -2.2189509868621826, + "logits/rejected": -2.2934834957122803, + "logps/chosen": -195.434326171875, + "logps/rejected": -185.47146606445312, + "loss": 1900.6648, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.06991083920001984, + "rewards/margins": 0.10115430504083633, + "rewards/rejected": -0.17106513679027557, + "step": 3270 + }, + { + "epoch": 0.21, + "learning_rate": 4.802728299485225e-06, + "logits/chosen": -2.082670211791992, + "logits/rejected": -2.0714988708496094, + "logps/chosen": -166.86862182617188, + "logps/rejected": -196.05703735351562, + "loss": 2415.0586, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.16264189779758453, + "rewards/margins": 0.07656769454479218, + "rewards/rejected": -0.2392096072435379, + "step": 3280 + }, + { + "epoch": 0.22, + "learning_rate": 4.8004991611308495e-06, + "logits/chosen": -2.3655083179473877, + "logits/rejected": -2.0764927864074707, + "logps/chosen": -250.82284545898438, + "logps/rejected": -250.86312866210938, + "loss": 1605.4111, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09269125759601593, + "rewards/margins": 0.16090475022792816, + "rewards/rejected": -0.2535960078239441, + "step": 3290 + }, + { + "epoch": 0.22, + "learning_rate": 4.798258022245937e-06, + "logits/chosen": -2.2947518825531006, + "logits/rejected": -1.8997490406036377, + "logps/chosen": -232.28384399414062, + "logps/rejected": -207.6372833251953, + "loss": 1823.3514, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1467735767364502, + "rewards/margins": 0.13818533718585968, + "rewards/rejected": -0.28495892882347107, + "step": 3300 + }, + { + "epoch": 0.22, + "eval_logits/chosen": -2.2288150787353516, + "eval_logits/rejected": -2.0495386123657227, + "eval_logps/chosen": -245.45086669921875, + "eval_logps/rejected": -238.97447204589844, + "eval_loss": 1848.311279296875, + "eval_rewards/accuracies": 0.6430000066757202, + "eval_rewards/chosen": -0.13445906341075897, + "eval_rewards/margins": 0.13916657865047455, + "eval_rewards/rejected": -0.2736256420612335, + "eval_runtime": 723.963, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.381, + "step": 3300 + }, + { + "epoch": 0.22, + "learning_rate": 4.796004894521365e-06, + "logits/chosen": -2.217214345932007, + "logits/rejected": -2.036797285079956, + "logps/chosen": -241.6744842529297, + "logps/rejected": -277.825439453125, + "loss": 1920.133, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.13104744255542755, + "rewards/margins": 0.1489441692829132, + "rewards/rejected": -0.27999162673950195, + "step": 3310 + }, + { + "epoch": 0.22, + "learning_rate": 4.7937397897105545e-06, + "logits/chosen": -2.206418514251709, + "logits/rejected": -2.125432014465332, + "logps/chosen": -219.0151824951172, + "logps/rejected": -203.31431579589844, + "loss": 2123.0996, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.13782355189323425, + "rewards/margins": 0.09512574225664139, + "rewards/rejected": -0.23294928669929504, + "step": 3320 + }, + { + "epoch": 0.22, + "learning_rate": 4.791462719629399e-06, + "logits/chosen": -2.217872142791748, + "logits/rejected": -2.0785470008850098, + "logps/chosen": -197.8411865234375, + "logps/rejected": -189.520263671875, + "loss": 1871.5398, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12116026878356934, + "rewards/margins": 0.1409054398536682, + "rewards/rejected": -0.26206570863723755, + "step": 3330 + }, + { + "epoch": 0.22, + "learning_rate": 4.789173696156212e-06, + "logits/chosen": -2.224691390991211, + "logits/rejected": -1.8815933465957642, + "logps/chosen": -290.6009826660156, + "logps/rejected": -292.05194091796875, + "loss": 1376.5012, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1416037529706955, + "rewards/margins": 0.201004296541214, + "rewards/rejected": -0.3426080346107483, + "step": 3340 + }, + { + "epoch": 0.22, + "learning_rate": 4.786872731231662e-06, + "logits/chosen": -2.2597310543060303, + "logits/rejected": -2.1441400051116943, + "logps/chosen": -237.48989868164062, + "logps/rejected": -237.34518432617188, + "loss": 1916.399, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.20598940551280975, + "rewards/margins": 0.11224385350942612, + "rewards/rejected": -0.31823328137397766, + "step": 3350 + }, + { + "epoch": 0.22, + "learning_rate": 4.784559836858709e-06, + "logits/chosen": -2.252401351928711, + "logits/rejected": -1.7922512292861938, + "logps/chosen": -249.43563842773438, + "logps/rejected": -234.86087036132812, + "loss": 1483.0201, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.15518534183502197, + "rewards/margins": 0.1584618091583252, + "rewards/rejected": -0.3136471211910248, + "step": 3360 + }, + { + "epoch": 0.22, + "learning_rate": 4.782235025102542e-06, + "logits/chosen": -2.2455966472625732, + "logits/rejected": -2.182466983795166, + "logps/chosen": -240.5248260498047, + "logps/rejected": -238.02346801757812, + "loss": 1967.6418, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.14205512404441833, + "rewards/margins": 0.14979180693626404, + "rewards/rejected": -0.2918469309806824, + "step": 3370 + }, + { + "epoch": 0.22, + "learning_rate": 4.779898308090519e-06, + "logits/chosen": -2.2079274654388428, + "logits/rejected": -1.9989955425262451, + "logps/chosen": -286.9621276855469, + "logps/rejected": -267.46014404296875, + "loss": 1900.592, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12284872680902481, + "rewards/margins": 0.17328739166259766, + "rewards/rejected": -0.2961360812187195, + "step": 3380 + }, + { + "epoch": 0.22, + "learning_rate": 4.777549698012101e-06, + "logits/chosen": -2.1511409282684326, + "logits/rejected": -1.9988940954208374, + "logps/chosen": -259.1100158691406, + "logps/rejected": -258.4521179199219, + "loss": 1776.992, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15893355011940002, + "rewards/margins": 0.1850178986787796, + "rewards/rejected": -0.34395143389701843, + "step": 3390 + }, + { + "epoch": 0.22, + "learning_rate": 4.775189207118787e-06, + "logits/chosen": -2.18037748336792, + "logits/rejected": -1.9939063787460327, + "logps/chosen": -288.7840270996094, + "logps/rejected": -281.2822265625, + "loss": 1617.63, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1696634292602539, + "rewards/margins": 0.14835859835147858, + "rewards/rejected": -0.3180220425128937, + "step": 3400 + }, + { + "epoch": 0.22, + "eval_logits/chosen": -2.2290878295898438, + "eval_logits/rejected": -2.049717903137207, + "eval_logps/chosen": -249.02952575683594, + "eval_logps/rejected": -242.2179718017578, + "eval_loss": 1866.488525390625, + "eval_rewards/accuracies": 0.6414999961853027, + "eval_rewards/chosen": -0.1702459454536438, + "eval_rewards/margins": 0.1358148157596588, + "eval_rewards/rejected": -0.3060607314109802, + "eval_runtime": 723.452, + "eval_samples_per_second": 2.765, + "eval_steps_per_second": 1.382, + "step": 3400 + }, + { + "epoch": 0.22, + "learning_rate": 4.772816847724054e-06, + "logits/chosen": -2.3193163871765137, + "logits/rejected": -2.0606117248535156, + "logps/chosen": -241.829833984375, + "logps/rejected": -251.56503295898438, + "loss": 2431.6721, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1927526593208313, + "rewards/margins": 0.10237501561641693, + "rewards/rejected": -0.2951276898384094, + "step": 3410 + }, + { + "epoch": 0.22, + "learning_rate": 4.770432632203294e-06, + "logits/chosen": -2.0573058128356934, + "logits/rejected": -1.9737217426300049, + "logps/chosen": -270.8164978027344, + "logps/rejected": -226.3074493408203, + "loss": 2336.4207, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.27169421315193176, + "rewards/margins": 0.05809188634157181, + "rewards/rejected": -0.3297860622406006, + "step": 3420 + }, + { + "epoch": 0.22, + "learning_rate": 4.768036572993738e-06, + "logits/chosen": -2.1174683570861816, + "logits/rejected": -2.2044284343719482, + "logps/chosen": -303.7652282714844, + "logps/rejected": -298.9791259765625, + "loss": 2009.1703, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.23746299743652344, + "rewards/margins": 0.10813780128955841, + "rewards/rejected": -0.34560078382492065, + "step": 3430 + }, + { + "epoch": 0.23, + "learning_rate": 4.765628682594409e-06, + "logits/chosen": -2.297766923904419, + "logits/rejected": -2.1239116191864014, + "logps/chosen": -262.238037109375, + "logps/rejected": -249.93252563476562, + "loss": 1776.9062, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1848095953464508, + "rewards/margins": 0.11596927791833878, + "rewards/rejected": -0.3007788360118866, + "step": 3440 + }, + { + "epoch": 0.23, + "learning_rate": 4.763208973566041e-06, + "logits/chosen": -2.104485034942627, + "logits/rejected": -2.1253323554992676, + "logps/chosen": -205.01254272460938, + "logps/rejected": -235.3104248046875, + "loss": 1871.548, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.20882877707481384, + "rewards/margins": 0.11582591384649277, + "rewards/rejected": -0.3246546685695648, + "step": 3450 + }, + { + "epoch": 0.23, + "learning_rate": 4.76077745853102e-06, + "logits/chosen": -2.296905755996704, + "logits/rejected": -2.159364700317383, + "logps/chosen": -283.6266174316406, + "logps/rejected": -297.8143615722656, + "loss": 2050.5668, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.33572444319725037, + "rewards/margins": 0.13062633574008942, + "rewards/rejected": -0.4663507342338562, + "step": 3460 + }, + { + "epoch": 0.23, + "learning_rate": 4.758334150173322e-06, + "logits/chosen": -2.139819622039795, + "logits/rejected": -1.965150237083435, + "logps/chosen": -287.6877746582031, + "logps/rejected": -279.01116943359375, + "loss": 1550.6515, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.24603800475597382, + "rewards/margins": 0.16377019882202148, + "rewards/rejected": -0.4098082184791565, + "step": 3470 + }, + { + "epoch": 0.23, + "learning_rate": 4.755879061238439e-06, + "logits/chosen": -2.2042808532714844, + "logits/rejected": -1.9885804653167725, + "logps/chosen": -289.332275390625, + "logps/rejected": -292.26824951171875, + "loss": 2013.2613, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3315892219543457, + "rewards/margins": 0.15573135018348694, + "rewards/rejected": -0.48732057213783264, + "step": 3480 + }, + { + "epoch": 0.23, + "learning_rate": 4.753412204533317e-06, + "logits/chosen": -2.364527702331543, + "logits/rejected": -1.8919954299926758, + "logps/chosen": -292.3900451660156, + "logps/rejected": -264.13616943359375, + "loss": 1665.785, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.29531973600387573, + "rewards/margins": 0.17137546837329865, + "rewards/rejected": -0.4666951596736908, + "step": 3490 + }, + { + "epoch": 0.23, + "learning_rate": 4.750933592926292e-06, + "logits/chosen": -2.2476608753204346, + "logits/rejected": -1.9270604848861694, + "logps/chosen": -255.2253875732422, + "logps/rejected": -246.07723999023438, + "loss": 1438.7666, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.36497727036476135, + "rewards/margins": 0.18068274855613708, + "rewards/rejected": -0.545660138130188, + "step": 3500 + }, + { + "epoch": 0.23, + "eval_logits/chosen": -2.1436102390289307, + "eval_logits/rejected": -1.970076322555542, + "eval_logps/chosen": -263.0986022949219, + "eval_logps/rejected": -256.9674072265625, + "eval_loss": 1877.9581298828125, + "eval_rewards/accuracies": 0.6470000147819519, + "eval_rewards/chosen": -0.31093651056289673, + "eval_rewards/margins": 0.14261873066425323, + "eval_rewards/rejected": -0.45355525612831116, + "eval_runtime": 724.8549, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 3500 + }, + { + "epoch": 0.23, + "learning_rate": 4.7484432393470124e-06, + "logits/chosen": -2.3309848308563232, + "logits/rejected": -1.824968695640564, + "logps/chosen": -228.2218475341797, + "logps/rejected": -190.5856475830078, + "loss": 1673.3477, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.26139897108078003, + "rewards/margins": 0.17760643362998962, + "rewards/rejected": -0.43900543451309204, + "step": 3510 + }, + { + "epoch": 0.23, + "learning_rate": 4.745941156786385e-06, + "logits/chosen": -1.9236522912979126, + "logits/rejected": -1.9156990051269531, + "logps/chosen": -181.29397583007812, + "logps/rejected": -226.3966522216797, + "loss": 1797.2092, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.28254860639572144, + "rewards/margins": 0.14261487126350403, + "rewards/rejected": -0.42516350746154785, + "step": 3520 + }, + { + "epoch": 0.23, + "learning_rate": 4.743427358296497e-06, + "logits/chosen": -2.053074359893799, + "logits/rejected": -1.8683373928070068, + "logps/chosen": -209.05892944335938, + "logps/rejected": -242.22012329101562, + "loss": 1428.9707, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1935068666934967, + "rewards/margins": 0.21853800117969513, + "rewards/rejected": -0.41204482316970825, + "step": 3530 + }, + { + "epoch": 0.23, + "learning_rate": 4.740901856990553e-06, + "logits/chosen": -1.9476228952407837, + "logits/rejected": -1.7711150646209717, + "logps/chosen": -279.8335266113281, + "logps/rejected": -244.2646026611328, + "loss": 2220.4473, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.23141586780548096, + "rewards/margins": 0.08503684401512146, + "rewards/rejected": -0.31645268201828003, + "step": 3540 + }, + { + "epoch": 0.23, + "learning_rate": 4.738364666042804e-06, + "logits/chosen": -2.169304132461548, + "logits/rejected": -1.770307183265686, + "logps/chosen": -307.50030517578125, + "logps/rejected": -260.1822814941406, + "loss": 1543.7248, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15755341947078705, + "rewards/margins": 0.16321787238121033, + "rewards/rejected": -0.32077130675315857, + "step": 3550 + }, + { + "epoch": 0.23, + "learning_rate": 4.735815798688483e-06, + "logits/chosen": -2.104562282562256, + "logits/rejected": -1.8917967081069946, + "logps/chosen": -219.07351684570312, + "logps/rejected": -255.24197387695312, + "loss": 1848.7328, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.2074153870344162, + "rewards/margins": 0.1602933704853058, + "rewards/rejected": -0.36770880222320557, + "step": 3560 + }, + { + "epoch": 0.23, + "learning_rate": 4.7332552682237285e-06, + "logits/chosen": -2.1062798500061035, + "logits/rejected": -1.6887365579605103, + "logps/chosen": -188.10646057128906, + "logps/rejected": -194.13392639160156, + "loss": 1267.9649, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14346513152122498, + "rewards/margins": 0.21799930930137634, + "rewards/rejected": -0.3614644408226013, + "step": 3570 + }, + { + "epoch": 0.23, + "learning_rate": 4.7306830880055234e-06, + "logits/chosen": -2.1045773029327393, + "logits/rejected": -2.0392093658447266, + "logps/chosen": -217.57101440429688, + "logps/rejected": -236.7305908203125, + "loss": 2163.6355, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2683719992637634, + "rewards/margins": 0.10822489112615585, + "rewards/rejected": -0.3765968680381775, + "step": 3580 + }, + { + "epoch": 0.23, + "learning_rate": 4.728099271451619e-06, + "logits/chosen": -2.129319190979004, + "logits/rejected": -2.060136318206787, + "logps/chosen": -212.7565155029297, + "logps/rejected": -218.15103149414062, + "loss": 1891.9111, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.21748504042625427, + "rewards/margins": 0.12347017228603363, + "rewards/rejected": -0.3409552276134491, + "step": 3590 + }, + { + "epoch": 0.24, + "learning_rate": 4.725503832040466e-06, + "logits/chosen": -1.9385948181152344, + "logits/rejected": -1.9002023935317993, + "logps/chosen": -169.0155487060547, + "logps/rejected": -205.9110565185547, + "loss": 2120.1508, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.198576882481575, + "rewards/margins": 0.10808098316192627, + "rewards/rejected": -0.3066578805446625, + "step": 3600 + }, + { + "epoch": 0.24, + "eval_logits/chosen": -2.0810132026672363, + "eval_logits/rejected": -1.910104513168335, + "eval_logps/chosen": -255.11273193359375, + "eval_logps/rejected": -248.61317443847656, + "eval_loss": 1857.2381591796875, + "eval_rewards/accuracies": 0.6504999995231628, + "eval_rewards/chosen": -0.23107774555683136, + "eval_rewards/margins": 0.1389349102973938, + "eval_rewards/rejected": -0.37001264095306396, + "eval_runtime": 726.2968, + "eval_samples_per_second": 2.754, + "eval_steps_per_second": 1.377, + "step": 3600 + }, + { + "epoch": 0.24, + "learning_rate": 4.722896783311152e-06, + "logits/chosen": -2.076775074005127, + "logits/rejected": -1.9659488201141357, + "logps/chosen": -285.59515380859375, + "logps/rejected": -348.5102233886719, + "loss": 2152.6373, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2654615044593811, + "rewards/margins": 0.11781106889247894, + "rewards/rejected": -0.38327255845069885, + "step": 3610 + }, + { + "epoch": 0.24, + "learning_rate": 4.720278138863318e-06, + "logits/chosen": -2.2312514781951904, + "logits/rejected": -2.0606610774993896, + "logps/chosen": -212.52163696289062, + "logps/rejected": -191.35154724121094, + "loss": 2192.3848, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.22399914264678955, + "rewards/margins": 0.106370709836483, + "rewards/rejected": -0.33036983013153076, + "step": 3620 + }, + { + "epoch": 0.24, + "learning_rate": 4.717647912357095e-06, + "logits/chosen": -2.2318344116210938, + "logits/rejected": -2.3128485679626465, + "logps/chosen": -297.67083740234375, + "logps/rejected": -313.06500244140625, + "loss": 2577.318, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.25193700194358826, + "rewards/margins": 0.024617036804556847, + "rewards/rejected": -0.27655404806137085, + "step": 3630 + }, + { + "epoch": 0.24, + "learning_rate": 4.715006117513035e-06, + "logits/chosen": -2.3442459106445312, + "logits/rejected": -2.1202805042266846, + "logps/chosen": -340.41632080078125, + "logps/rejected": -299.45916748046875, + "loss": 1648.0377, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1629071980714798, + "rewards/margins": 0.14222896099090576, + "rewards/rejected": -0.30513614416122437, + "step": 3640 + }, + { + "epoch": 0.24, + "learning_rate": 4.7123527681120326e-06, + "logits/chosen": -2.16243052482605, + "logits/rejected": -2.0179338455200195, + "logps/chosen": -266.94195556640625, + "logps/rejected": -250.9875946044922, + "loss": 1774.2609, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.197707861661911, + "rewards/margins": 0.12197317183017731, + "rewards/rejected": -0.3196810781955719, + "step": 3650 + }, + { + "epoch": 0.24, + "learning_rate": 4.7096878779952594e-06, + "logits/chosen": -2.2487692832946777, + "logits/rejected": -2.1916444301605225, + "logps/chosen": -296.69451904296875, + "logps/rejected": -311.2198791503906, + "loss": 1489.8128, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.21281103789806366, + "rewards/margins": 0.17108117043972015, + "rewards/rejected": -0.3838922083377838, + "step": 3660 + }, + { + "epoch": 0.24, + "learning_rate": 4.707011461064086e-06, + "logits/chosen": -2.036893844604492, + "logits/rejected": -1.7952464818954468, + "logps/chosen": -326.61944580078125, + "logps/rejected": -300.3973083496094, + "loss": 1669.2322, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.17760398983955383, + "rewards/margins": 0.17968544363975525, + "rewards/rejected": -0.35728946328163147, + "step": 3670 + }, + { + "epoch": 0.24, + "learning_rate": 4.704323531280016e-06, + "logits/chosen": -2.0617146492004395, + "logits/rejected": -1.9116485118865967, + "logps/chosen": -344.8534240722656, + "logps/rejected": -277.119140625, + "loss": 1866.4686, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.17641881108283997, + "rewards/margins": 0.1546126902103424, + "rewards/rejected": -0.3310315012931824, + "step": 3680 + }, + { + "epoch": 0.24, + "learning_rate": 4.701624102664606e-06, + "logits/chosen": -2.2474734783172607, + "logits/rejected": -1.929483413696289, + "logps/chosen": -283.1015625, + "logps/rejected": -239.931640625, + "loss": 2034.3928, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.21985697746276855, + "rewards/margins": 0.10930268466472626, + "rewards/rejected": -0.329159677028656, + "step": 3690 + }, + { + "epoch": 0.24, + "learning_rate": 4.698913189299399e-06, + "logits/chosen": -2.0997631549835205, + "logits/rejected": -2.1936910152435303, + "logps/chosen": -202.63076782226562, + "logps/rejected": -245.99765014648438, + "loss": 2384.8625, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.16478955745697021, + "rewards/margins": 0.10860179364681244, + "rewards/rejected": -0.27339136600494385, + "step": 3700 + }, + { + "epoch": 0.24, + "eval_logits/chosen": -2.1779725551605225, + "eval_logits/rejected": -2.0020358562469482, + "eval_logps/chosen": -247.79246520996094, + "eval_logps/rejected": -241.4032745361328, + "eval_loss": 1852.2376708984375, + "eval_rewards/accuracies": 0.6499999761581421, + "eval_rewards/chosen": -0.15787512063980103, + "eval_rewards/margins": 0.1400386244058609, + "eval_rewards/rejected": -0.29791373014450073, + "eval_runtime": 724.8438, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 3700 + }, + { + "epoch": 0.24, + "learning_rate": 4.696190805325847e-06, + "logits/chosen": -2.2065725326538086, + "logits/rejected": -2.0684335231781006, + "logps/chosen": -220.91055297851562, + "logps/rejected": -209.87887573242188, + "loss": 1673.4158, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14531712234020233, + "rewards/margins": 0.16087010502815247, + "rewards/rejected": -0.306187242269516, + "step": 3710 + }, + { + "epoch": 0.24, + "learning_rate": 4.693456964945239e-06, + "logits/chosen": -2.320401191711426, + "logits/rejected": -1.8658478260040283, + "logps/chosen": -307.8102722167969, + "logps/rejected": -225.3717041015625, + "loss": 1202.8632, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0970473513007164, + "rewards/margins": 0.18653647601604462, + "rewards/rejected": -0.2835838198661804, + "step": 3720 + }, + { + "epoch": 0.24, + "learning_rate": 4.6907116824186245e-06, + "logits/chosen": -2.2743570804595947, + "logits/rejected": -2.231112241744995, + "logps/chosen": -236.314697265625, + "logps/rejected": -245.04940795898438, + "loss": 2315.3547, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.08987172693014145, + "rewards/margins": 0.08930940926074982, + "rewards/rejected": -0.17918114364147186, + "step": 3730 + }, + { + "epoch": 0.24, + "learning_rate": 4.687954972066742e-06, + "logits/chosen": -2.1849050521850586, + "logits/rejected": -1.9289398193359375, + "logps/chosen": -239.7222137451172, + "logps/rejected": -239.80740356445312, + "loss": 1672.2711, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09905945509672165, + "rewards/margins": 0.2095617949962616, + "rewards/rejected": -0.30862125754356384, + "step": 3740 + }, + { + "epoch": 0.25, + "learning_rate": 4.685186848269944e-06, + "logits/chosen": -2.1947669982910156, + "logits/rejected": -2.0647358894348145, + "logps/chosen": -222.42724609375, + "logps/rejected": -196.56259155273438, + "loss": 1854.6092, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10554896295070648, + "rewards/margins": 0.1151428073644638, + "rewards/rejected": -0.2206917703151703, + "step": 3750 + }, + { + "epoch": 0.25, + "learning_rate": 4.682407325468119e-06, + "logits/chosen": -2.277951717376709, + "logits/rejected": -1.9266493320465088, + "logps/chosen": -225.47329711914062, + "logps/rejected": -207.5655517578125, + "loss": 1687.2664, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08054288476705551, + "rewards/margins": 0.15721990168094635, + "rewards/rejected": -0.23776277899742126, + "step": 3760 + }, + { + "epoch": 0.25, + "learning_rate": 4.67961641816062e-06, + "logits/chosen": -2.248936176300049, + "logits/rejected": -2.052278757095337, + "logps/chosen": -284.053955078125, + "logps/rejected": -254.1475067138672, + "loss": 2078.1754, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.08257193863391876, + "rewards/margins": 0.1142364963889122, + "rewards/rejected": -0.19680842757225037, + "step": 3770 + }, + { + "epoch": 0.25, + "learning_rate": 4.676814140906188e-06, + "logits/chosen": -2.1284399032592773, + "logits/rejected": -2.001819133758545, + "logps/chosen": -254.8821258544922, + "logps/rejected": -239.061279296875, + "loss": 1785.2656, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15345919132232666, + "rewards/margins": 0.14784851670265198, + "rewards/rejected": -0.301307737827301, + "step": 3780 + }, + { + "epoch": 0.25, + "learning_rate": 4.674000508322872e-06, + "logits/chosen": -1.996980905532837, + "logits/rejected": -2.069845199584961, + "logps/chosen": -233.3638458251953, + "logps/rejected": -250.4616241455078, + "loss": 2388.3703, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.15828964114189148, + "rewards/margins": 0.07914143055677414, + "rewards/rejected": -0.23743107914924622, + "step": 3790 + }, + { + "epoch": 0.25, + "learning_rate": 4.671175535087959e-06, + "logits/chosen": -2.1575369834899902, + "logits/rejected": -2.1205859184265137, + "logps/chosen": -308.8890075683594, + "logps/rejected": -327.61541748046875, + "loss": 1321.1524, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.17561353743076324, + "rewards/margins": 0.21958065032958984, + "rewards/rejected": -0.3951941728591919, + "step": 3800 + }, + { + "epoch": 0.25, + "eval_logits/chosen": -2.2452280521392822, + "eval_logits/rejected": -2.065730571746826, + "eval_logps/chosen": -247.39849853515625, + "eval_logps/rejected": -239.1979217529297, + "eval_loss": 1872.46875, + "eval_rewards/accuracies": 0.6470000147819519, + "eval_rewards/chosen": -0.15393544733524323, + "eval_rewards/margins": 0.12192466855049133, + "eval_rewards/rejected": -0.27586010098457336, + "eval_runtime": 724.0232, + "eval_samples_per_second": 2.762, + "eval_steps_per_second": 1.381, + "step": 3800 + }, + { + "epoch": 0.25, + "learning_rate": 4.6683392359378924e-06, + "logits/chosen": -2.1318156719207764, + "logits/rejected": -1.9592097997665405, + "logps/chosen": -247.49972534179688, + "logps/rejected": -237.6021270751953, + "loss": 1829.1955, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12350733578205109, + "rewards/margins": 0.15618854761123657, + "rewards/rejected": -0.27969586849212646, + "step": 3810 + }, + { + "epoch": 0.25, + "learning_rate": 4.665491625668198e-06, + "logits/chosen": -2.059387683868408, + "logits/rejected": -2.105896472930908, + "logps/chosen": -172.2931671142578, + "logps/rejected": -207.222412109375, + "loss": 2150.1033, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.16045895218849182, + "rewards/margins": 0.11374203115701675, + "rewards/rejected": -0.274200975894928, + "step": 3820 + }, + { + "epoch": 0.25, + "learning_rate": 4.662632719133407e-06, + "logits/chosen": -2.342655658721924, + "logits/rejected": -2.0594210624694824, + "logps/chosen": -238.0370635986328, + "logps/rejected": -183.14178466796875, + "loss": 1867.0152, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.07239507883787155, + "rewards/margins": 0.12112174183130264, + "rewards/rejected": -0.1935168206691742, + "step": 3830 + }, + { + "epoch": 0.25, + "learning_rate": 4.659762531246974e-06, + "logits/chosen": -2.2439637184143066, + "logits/rejected": -2.091200590133667, + "logps/chosen": -231.6832275390625, + "logps/rejected": -206.22177124023438, + "loss": 2122.8434, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17104485630989075, + "rewards/margins": 0.08495251089334488, + "rewards/rejected": -0.25599735975265503, + "step": 3840 + }, + { + "epoch": 0.25, + "learning_rate": 4.656881076981207e-06, + "logits/chosen": -2.2985904216766357, + "logits/rejected": -2.1653382778167725, + "logps/chosen": -224.47488403320312, + "logps/rejected": -211.1822052001953, + "loss": 2243.6316, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.11291512101888657, + "rewards/margins": 0.046060457825660706, + "rewards/rejected": -0.15897558629512787, + "step": 3850 + }, + { + "epoch": 0.25, + "learning_rate": 4.653988371367183e-06, + "logits/chosen": -2.231370449066162, + "logits/rejected": -1.9748144149780273, + "logps/chosen": -251.9241485595703, + "logps/rejected": -203.20687866210938, + "loss": 1686.8398, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11120152473449707, + "rewards/margins": 0.1249074935913086, + "rewards/rejected": -0.23610901832580566, + "step": 3860 + }, + { + "epoch": 0.25, + "learning_rate": 4.651084429494671e-06, + "logits/chosen": -2.2892768383026123, + "logits/rejected": -2.0120041370391846, + "logps/chosen": -288.3143005371094, + "logps/rejected": -224.89505004882812, + "loss": 1673.5797, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.12963344156742096, + "rewards/margins": 0.1610449105501175, + "rewards/rejected": -0.29067832231521606, + "step": 3870 + }, + { + "epoch": 0.25, + "learning_rate": 4.648169266512053e-06, + "logits/chosen": -2.3402390480041504, + "logits/rejected": -2.069182872772217, + "logps/chosen": -234.9876251220703, + "logps/rejected": -200.89300537109375, + "loss": 1949.3049, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09659501165151596, + "rewards/margins": 0.10859502851963043, + "rewards/rejected": -0.2051900327205658, + "step": 3880 + }, + { + "epoch": 0.25, + "learning_rate": 4.6452428976262505e-06, + "logits/chosen": -2.149710178375244, + "logits/rejected": -1.9293615818023682, + "logps/chosen": -209.88943481445312, + "logps/rejected": -186.5958709716797, + "loss": 1336.7575, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.06097991019487381, + "rewards/margins": 0.21029214560985565, + "rewards/rejected": -0.27127203345298767, + "step": 3890 + }, + { + "epoch": 0.26, + "learning_rate": 4.642305338102633e-06, + "logits/chosen": -2.1881473064422607, + "logits/rejected": -2.272202968597412, + "logps/chosen": -169.59375, + "logps/rejected": -199.1464385986328, + "loss": 1975.567, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.07844208925962448, + "rewards/margins": 0.11406981945037842, + "rewards/rejected": -0.1925118863582611, + "step": 3900 + }, + { + "epoch": 0.26, + "eval_logits/chosen": -2.211078643798828, + "eval_logits/rejected": -2.0324509143829346, + "eval_logps/chosen": -237.15188598632812, + "eval_logps/rejected": -232.16534423828125, + "eval_loss": 1848.4305419921875, + "eval_rewards/accuracies": 0.6554999947547913, + "eval_rewards/chosen": -0.05146953463554382, + "eval_rewards/margins": 0.15406478941440582, + "eval_rewards/rejected": -0.20553433895111084, + "eval_runtime": 725.9243, + "eval_samples_per_second": 2.755, + "eval_steps_per_second": 1.378, + "step": 3900 + }, + { + "epoch": 0.26, + "learning_rate": 4.639356603264953e-06, + "logits/chosen": -2.2523646354675293, + "logits/rejected": -2.032790184020996, + "logps/chosen": -248.07278442382812, + "logps/rejected": -236.7919921875, + "loss": 2005.1246, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.062409840524196625, + "rewards/margins": 0.14450041949748993, + "rewards/rejected": -0.20691028237342834, + "step": 3910 + }, + { + "epoch": 0.26, + "learning_rate": 4.636396708495255e-06, + "logits/chosen": -2.09509015083313, + "logits/rejected": -2.0792410373687744, + "logps/chosen": -234.39501953125, + "logps/rejected": -227.2949676513672, + "loss": 1513.6564, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.05199925974011421, + "rewards/margins": 0.17743878066539764, + "rewards/rejected": -0.22943802177906036, + "step": 3920 + }, + { + "epoch": 0.26, + "learning_rate": 4.633425669233799e-06, + "logits/chosen": -2.2118465900421143, + "logits/rejected": -2.2278549671173096, + "logps/chosen": -244.0120086669922, + "logps/rejected": -249.1499786376953, + "loss": 2088.2393, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09185406565666199, + "rewards/margins": 0.09585288166999817, + "rewards/rejected": -0.18770697712898254, + "step": 3930 + }, + { + "epoch": 0.26, + "learning_rate": 4.6304435009789825e-06, + "logits/chosen": -2.2175028324127197, + "logits/rejected": -1.9971988201141357, + "logps/chosen": -243.27426147460938, + "logps/rejected": -186.73477172851562, + "loss": 1765.7463, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.05767058581113815, + "rewards/margins": 0.14370371401309967, + "rewards/rejected": -0.20137432217597961, + "step": 3940 + }, + { + "epoch": 0.26, + "learning_rate": 4.627450219287256e-06, + "logits/chosen": -2.2618629932403564, + "logits/rejected": -2.1243340969085693, + "logps/chosen": -192.6259765625, + "logps/rejected": -181.14503479003906, + "loss": 2074.3313, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11066880077123642, + "rewards/margins": 0.1103011816740036, + "rewards/rejected": -0.22096994519233704, + "step": 3950 + }, + { + "epoch": 0.26, + "learning_rate": 4.624445839773042e-06, + "logits/chosen": -2.2068986892700195, + "logits/rejected": -2.1380820274353027, + "logps/chosen": -184.40818786621094, + "logps/rejected": -188.77137756347656, + "loss": 2496.201, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.1439688503742218, + "rewards/margins": 0.05427010729908943, + "rewards/rejected": -0.19823895394802094, + "step": 3960 + }, + { + "epoch": 0.26, + "learning_rate": 4.621430378108656e-06, + "logits/chosen": -2.197033405303955, + "logits/rejected": -2.040515184402466, + "logps/chosen": -273.7949523925781, + "logps/rejected": -280.93048095703125, + "loss": 1702.4613, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1603659689426422, + "rewards/margins": 0.13899901509284973, + "rewards/rejected": -0.29936498403549194, + "step": 3970 + }, + { + "epoch": 0.26, + "learning_rate": 4.618403850024223e-06, + "logits/chosen": -2.104780435562134, + "logits/rejected": -1.8865197896957397, + "logps/chosen": -268.41644287109375, + "logps/rejected": -232.78341674804688, + "loss": 1980.4014, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.129262313246727, + "rewards/margins": 0.11029335111379623, + "rewards/rejected": -0.23955564200878143, + "step": 3980 + }, + { + "epoch": 0.26, + "learning_rate": 4.615366271307598e-06, + "logits/chosen": -2.266728162765503, + "logits/rejected": -2.113433361053467, + "logps/chosen": -208.0624542236328, + "logps/rejected": -208.528076171875, + "loss": 1785.8047, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16119475662708282, + "rewards/margins": 0.12357983738183975, + "rewards/rejected": -0.2847746014595032, + "step": 3990 + }, + { + "epoch": 0.26, + "learning_rate": 4.612317657804277e-06, + "logits/chosen": -2.1275062561035156, + "logits/rejected": -2.176178455352783, + "logps/chosen": -163.8437042236328, + "logps/rejected": -229.2069854736328, + "loss": 1914.4258, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.16610661149024963, + "rewards/margins": 0.14015412330627441, + "rewards/rejected": -0.30626070499420166, + "step": 4000 + }, + { + "epoch": 0.26, + "eval_logits/chosen": -2.2187206745147705, + "eval_logits/rejected": -2.0401241779327393, + "eval_logps/chosen": -245.95481872558594, + "eval_logps/rejected": -239.28158569335938, + "eval_loss": 1862.8907470703125, + "eval_rewards/accuracies": 0.6535000205039978, + "eval_rewards/chosen": -0.13949863612651825, + "eval_rewards/margins": 0.13719816505908966, + "eval_rewards/rejected": -0.2766967713832855, + "eval_runtime": 727.2291, + "eval_samples_per_second": 2.75, + "eval_steps_per_second": 1.375, + "step": 4000 + }, + { + "epoch": 0.26, + "learning_rate": 4.6092580254173236e-06, + "logits/chosen": -2.1136960983276367, + "logits/rejected": -1.892932653427124, + "logps/chosen": -269.98236083984375, + "logps/rejected": -268.93701171875, + "loss": 1639.9719, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12659600377082825, + "rewards/margins": 0.17622780799865723, + "rewards/rejected": -0.3028238117694855, + "step": 4010 + }, + { + "epoch": 0.26, + "learning_rate": 4.606187390107277e-06, + "logits/chosen": -2.0953426361083984, + "logits/rejected": -1.9211162328720093, + "logps/chosen": -241.5983428955078, + "logps/rejected": -212.2313232421875, + "loss": 2033.3469, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.167229562997818, + "rewards/margins": 0.09625671803951263, + "rewards/rejected": -0.26348626613616943, + "step": 4020 + }, + { + "epoch": 0.26, + "learning_rate": 4.603105767892077e-06, + "logits/chosen": -2.2119300365448, + "logits/rejected": -2.1371030807495117, + "logps/chosen": -207.75601196289062, + "logps/rejected": -240.72824096679688, + "loss": 1638.3612, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.13065990805625916, + "rewards/margins": 0.15018993616104126, + "rewards/rejected": -0.2808498442173004, + "step": 4030 + }, + { + "epoch": 0.26, + "learning_rate": 4.6000131748469725e-06, + "logits/chosen": -2.27934193611145, + "logits/rejected": -1.9166730642318726, + "logps/chosen": -256.64068603515625, + "logps/rejected": -193.72409057617188, + "loss": 2078.6699, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.06500424444675446, + "rewards/margins": 0.11401742696762085, + "rewards/rejected": -0.1790216565132141, + "step": 4040 + }, + { + "epoch": 0.26, + "learning_rate": 4.596909627104445e-06, + "logits/chosen": -2.3300695419311523, + "logits/rejected": -2.251364231109619, + "logps/chosen": -260.0342102050781, + "logps/rejected": -241.5802764892578, + "loss": 2057.6164, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.11295942217111588, + "rewards/margins": 0.1418023258447647, + "rewards/rejected": -0.2547617554664612, + "step": 4050 + }, + { + "epoch": 0.27, + "learning_rate": 4.5937951408541215e-06, + "logits/chosen": -2.369083881378174, + "logits/rejected": -1.8532183170318604, + "logps/chosen": -262.75360107421875, + "logps/rejected": -235.9652099609375, + "loss": 1622.623, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10115152597427368, + "rewards/margins": 0.185886412858963, + "rewards/rejected": -0.2870379388332367, + "step": 4060 + }, + { + "epoch": 0.27, + "learning_rate": 4.590669732342685e-06, + "logits/chosen": -2.0819499492645264, + "logits/rejected": -1.943646788597107, + "logps/chosen": -220.8902130126953, + "logps/rejected": -239.7845001220703, + "loss": 1625.7482, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08742596954107285, + "rewards/margins": 0.1700166016817093, + "rewards/rejected": -0.25744256377220154, + "step": 4070 + }, + { + "epoch": 0.27, + "learning_rate": 4.587533417873799e-06, + "logits/chosen": -2.1543188095092773, + "logits/rejected": -2.2358295917510986, + "logps/chosen": -207.8849334716797, + "logps/rejected": -284.0242614746094, + "loss": 1614.4966, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14720898866653442, + "rewards/margins": 0.16435673832893372, + "rewards/rejected": -0.31156572699546814, + "step": 4080 + }, + { + "epoch": 0.27, + "learning_rate": 4.584386213808016e-06, + "logits/chosen": -2.1320371627807617, + "logits/rejected": -1.815946340560913, + "logps/chosen": -235.2030792236328, + "logps/rejected": -201.69064331054688, + "loss": 1804.5004, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.13524064421653748, + "rewards/margins": 0.12263710796833038, + "rewards/rejected": -0.25787776708602905, + "step": 4090 + }, + { + "epoch": 0.27, + "learning_rate": 4.581228136562693e-06, + "logits/chosen": -2.03398060798645, + "logits/rejected": -2.144202947616577, + "logps/chosen": -250.9056854248047, + "logps/rejected": -231.795654296875, + "loss": 2663.7783, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1272524893283844, + "rewards/margins": 0.06574568152427673, + "rewards/rejected": -0.19299815595149994, + "step": 4100 + }, + { + "epoch": 0.27, + "eval_logits/chosen": -2.215273141860962, + "eval_logits/rejected": -2.0376996994018555, + "eval_logps/chosen": -247.2678680419922, + "eval_logps/rejected": -240.6035614013672, + "eval_loss": 1853.19189453125, + "eval_rewards/accuracies": 0.6535000205039978, + "eval_rewards/chosen": -0.15262925624847412, + "eval_rewards/margins": 0.13728740811347961, + "eval_rewards/rejected": -0.28991666436195374, + "eval_runtime": 724.4867, + "eval_samples_per_second": 2.761, + "eval_steps_per_second": 1.38, + "step": 4100 + }, + { + "epoch": 0.27, + "learning_rate": 4.578059202611909e-06, + "logits/chosen": -2.2305524349212646, + "logits/rejected": -2.020010471343994, + "logps/chosen": -270.2746276855469, + "logps/rejected": -265.7089538574219, + "loss": 2003.9635, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13115721940994263, + "rewards/margins": 0.10190287977457047, + "rewards/rejected": -0.2330600768327713, + "step": 4110 + }, + { + "epoch": 0.27, + "learning_rate": 4.574879428486376e-06, + "logits/chosen": -2.208253860473633, + "logits/rejected": -1.9521793127059937, + "logps/chosen": -231.3865203857422, + "logps/rejected": -243.73385620117188, + "loss": 1639.358, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.18469981849193573, + "rewards/margins": 0.15269534289836884, + "rewards/rejected": -0.33739519119262695, + "step": 4120 + }, + { + "epoch": 0.27, + "learning_rate": 4.571688830773352e-06, + "logits/chosen": -2.236798048019409, + "logits/rejected": -2.132211208343506, + "logps/chosen": -235.32858276367188, + "logps/rejected": -223.46475219726562, + "loss": 2082.0211, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1326039731502533, + "rewards/margins": 0.08015134930610657, + "rewards/rejected": -0.21275529265403748, + "step": 4130 + }, + { + "epoch": 0.27, + "learning_rate": 4.568487426116559e-06, + "logits/chosen": -2.1624183654785156, + "logits/rejected": -2.169710636138916, + "logps/chosen": -180.14920043945312, + "logps/rejected": -184.7618865966797, + "loss": 1673.3771, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.08177350461483002, + "rewards/margins": 0.12146095186471939, + "rewards/rejected": -0.20323446393013, + "step": 4140 + }, + { + "epoch": 0.27, + "learning_rate": 4.565275231216092e-06, + "logits/chosen": -2.066892385482788, + "logits/rejected": -2.044159412384033, + "logps/chosen": -160.7727813720703, + "logps/rejected": -217.7138671875, + "loss": 1986.8971, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.09429313987493515, + "rewards/margins": 0.1095115914940834, + "rewards/rejected": -0.20380473136901855, + "step": 4150 + }, + { + "epoch": 0.27, + "learning_rate": 4.562052262828331e-06, + "logits/chosen": -2.1357436180114746, + "logits/rejected": -2.003145456314087, + "logps/chosen": -213.689453125, + "logps/rejected": -223.7344207763672, + "loss": 1702.7379, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14878423511981964, + "rewards/margins": 0.15320990979671478, + "rewards/rejected": -0.3019941449165344, + "step": 4160 + }, + { + "epoch": 0.27, + "learning_rate": 4.558818537765861e-06, + "logits/chosen": -2.3047287464141846, + "logits/rejected": -2.0756564140319824, + "logps/chosen": -252.1100616455078, + "logps/rejected": -233.3250274658203, + "loss": 1776.3279, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16095681488513947, + "rewards/margins": 0.1608300507068634, + "rewards/rejected": -0.3217868506908417, + "step": 4170 + }, + { + "epoch": 0.27, + "learning_rate": 4.555574072897374e-06, + "logits/chosen": -2.1982951164245605, + "logits/rejected": -2.209923267364502, + "logps/chosen": -221.6260528564453, + "logps/rejected": -228.4404754638672, + "loss": 2324.7164, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.19664804637432098, + "rewards/margins": 0.09086047857999802, + "rewards/rejected": -0.2875085175037384, + "step": 4180 + }, + { + "epoch": 0.27, + "learning_rate": 4.552318885147589e-06, + "logits/chosen": -2.3210973739624023, + "logits/rejected": -1.977421760559082, + "logps/chosen": -255.15255737304688, + "logps/rejected": -209.5719451904297, + "loss": 1803.0857, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1423492282629013, + "rewards/margins": 0.13808949291706085, + "rewards/rejected": -0.28043872117996216, + "step": 4190 + }, + { + "epoch": 0.27, + "learning_rate": 4.549052991497159e-06, + "logits/chosen": -2.192139148712158, + "logits/rejected": -2.164456844329834, + "logps/chosen": -194.3700714111328, + "logps/rejected": -207.3929443359375, + "loss": 1816.1697, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14202898740768433, + "rewards/margins": 0.1260402500629425, + "rewards/rejected": -0.26806920766830444, + "step": 4200 + }, + { + "epoch": 0.27, + "eval_logits/chosen": -2.2034807205200195, + "eval_logits/rejected": -2.0271365642547607, + "eval_logps/chosen": -244.57276916503906, + "eval_logps/rejected": -238.06834411621094, + "eval_loss": 1847.70458984375, + "eval_rewards/accuracies": 0.656000018119812, + "eval_rewards/chosen": -0.1256781369447708, + "eval_rewards/margins": 0.13888615369796753, + "eval_rewards/rejected": -0.26456430554389954, + "eval_runtime": 725.6716, + "eval_samples_per_second": 2.756, + "eval_steps_per_second": 1.378, + "step": 4200 + }, + { + "epoch": 0.28, + "learning_rate": 4.545776408982585e-06, + "logits/chosen": -2.132502794265747, + "logits/rejected": -2.126312255859375, + "logps/chosen": -243.8193817138672, + "logps/rejected": -246.9946746826172, + "loss": 2001.8141, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.12075313180685043, + "rewards/margins": 0.12188541889190674, + "rewards/rejected": -0.2426385134458542, + "step": 4210 + }, + { + "epoch": 0.28, + "learning_rate": 4.542489154696128e-06, + "logits/chosen": -2.3441829681396484, + "logits/rejected": -1.9969489574432373, + "logps/chosen": -274.8600158691406, + "logps/rejected": -226.11526489257812, + "loss": 1901.0047, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.07139638811349869, + "rewards/margins": 0.12505993247032166, + "rewards/rejected": -0.19645631313323975, + "step": 4220 + }, + { + "epoch": 0.28, + "learning_rate": 4.5391912457857145e-06, + "logits/chosen": -2.2296595573425293, + "logits/rejected": -1.9845161437988281, + "logps/chosen": -274.591064453125, + "logps/rejected": -239.41983032226562, + "loss": 1906.3684, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09621410816907883, + "rewards/margins": 0.1193847805261612, + "rewards/rejected": -0.21559889614582062, + "step": 4230 + }, + { + "epoch": 0.28, + "learning_rate": 4.535882699454854e-06, + "logits/chosen": -2.233431339263916, + "logits/rejected": -2.115579128265381, + "logps/chosen": -284.89764404296875, + "logps/rejected": -325.75762939453125, + "loss": 1728.6193, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1384657323360443, + "rewards/margins": 0.191973477602005, + "rewards/rejected": -0.3304392099380493, + "step": 4240 + }, + { + "epoch": 0.28, + "learning_rate": 4.532563532962546e-06, + "logits/chosen": -2.297442674636841, + "logits/rejected": -2.3749196529388428, + "logps/chosen": -208.76651000976562, + "logps/rejected": -241.6005859375, + "loss": 2318.2254, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.188722625374794, + "rewards/margins": 0.11239129304885864, + "rewards/rejected": -0.30111393332481384, + "step": 4250 + }, + { + "epoch": 0.28, + "learning_rate": 4.529233763623187e-06, + "logits/chosen": -2.2471325397491455, + "logits/rejected": -1.9625120162963867, + "logps/chosen": -219.39297485351562, + "logps/rejected": -182.62008666992188, + "loss": 2061.3088, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17292466759681702, + "rewards/margins": 0.11186778545379639, + "rewards/rejected": -0.2847924828529358, + "step": 4260 + }, + { + "epoch": 0.28, + "learning_rate": 4.5258934088064854e-06, + "logits/chosen": -2.1585116386413574, + "logits/rejected": -1.7696845531463623, + "logps/chosen": -240.76730346679688, + "logps/rejected": -201.3708038330078, + "loss": 1758.8008, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.20786544680595398, + "rewards/margins": 0.15300999581813812, + "rewards/rejected": -0.3608754575252533, + "step": 4270 + }, + { + "epoch": 0.28, + "learning_rate": 4.522542485937369e-06, + "logits/chosen": -2.262988567352295, + "logits/rejected": -2.0149128437042236, + "logps/chosen": -307.56085205078125, + "logps/rejected": -229.0597381591797, + "loss": 1402.5003, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.1690836250782013, + "rewards/margins": 0.18139702081680298, + "rewards/rejected": -0.3504806458950043, + "step": 4280 + }, + { + "epoch": 0.28, + "learning_rate": 4.519181012495892e-06, + "logits/chosen": -2.273116111755371, + "logits/rejected": -2.140737295150757, + "logps/chosen": -256.5496826171875, + "logps/rejected": -243.6362762451172, + "loss": 1785.6244, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14330413937568665, + "rewards/margins": 0.1616070568561554, + "rewards/rejected": -0.30491116642951965, + "step": 4290 + }, + { + "epoch": 0.28, + "learning_rate": 4.515809006017147e-06, + "logits/chosen": -2.191171646118164, + "logits/rejected": -1.8993133306503296, + "logps/chosen": -248.03738403320312, + "logps/rejected": -223.3999481201172, + "loss": 2080.9717, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14627774059772491, + "rewards/margins": 0.1087186336517334, + "rewards/rejected": -0.2549963891506195, + "step": 4300 + }, + { + "epoch": 0.28, + "eval_logits/chosen": -2.2131552696228027, + "eval_logits/rejected": -2.0350379943847656, + "eval_logps/chosen": -246.4134979248047, + "eval_logps/rejected": -239.87435913085938, + "eval_loss": 1834.1217041015625, + "eval_rewards/accuracies": 0.6549999713897705, + "eval_rewards/chosen": -0.14408540725708008, + "eval_rewards/margins": 0.1385391652584076, + "eval_rewards/rejected": -0.28262457251548767, + "eval_runtime": 726.458, + "eval_samples_per_second": 2.753, + "eval_steps_per_second": 1.377, + "step": 4300 + }, + { + "epoch": 0.28, + "learning_rate": 4.512426484091171e-06, + "logits/chosen": -2.3461813926696777, + "logits/rejected": -2.020876169204712, + "logps/chosen": -290.72125244140625, + "logps/rejected": -267.656005859375, + "loss": 1668.8262, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13140076398849487, + "rewards/margins": 0.1322779804468155, + "rewards/rejected": -0.26367875933647156, + "step": 4310 + }, + { + "epoch": 0.28, + "learning_rate": 4.509033464362858e-06, + "logits/chosen": -2.053769111633301, + "logits/rejected": -2.112048864364624, + "logps/chosen": -252.5277099609375, + "logps/rejected": -284.105224609375, + "loss": 1808.401, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11261830478906631, + "rewards/margins": 0.17346562445163727, + "rewards/rejected": -0.2860839068889618, + "step": 4320 + }, + { + "epoch": 0.28, + "learning_rate": 4.505629964531857e-06, + "logits/chosen": -2.32875919342041, + "logits/rejected": -2.133932590484619, + "logps/chosen": -235.81735229492188, + "logps/rejected": -220.7907257080078, + "loss": 1688.3691, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1261296570301056, + "rewards/margins": 0.15786603093147278, + "rewards/rejected": -0.28399568796157837, + "step": 4330 + }, + { + "epoch": 0.28, + "learning_rate": 4.502216002352492e-06, + "logits/chosen": -2.3252675533294678, + "logits/rejected": -2.10353422164917, + "logps/chosen": -176.7300567626953, + "logps/rejected": -171.5137176513672, + "loss": 1939.9209, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1368914544582367, + "rewards/margins": 0.14197859168052673, + "rewards/rejected": -0.2788700461387634, + "step": 4340 + }, + { + "epoch": 0.28, + "learning_rate": 4.498791595633663e-06, + "logits/chosen": -2.1548573970794678, + "logits/rejected": -1.7888492345809937, + "logps/chosen": -275.2837829589844, + "logps/rejected": -200.15524291992188, + "loss": 1920.0037, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12899497151374817, + "rewards/margins": 0.12071913480758667, + "rewards/rejected": -0.24971409142017365, + "step": 4350 + }, + { + "epoch": 0.29, + "learning_rate": 4.495356762238751e-06, + "logits/chosen": -2.402125597000122, + "logits/rejected": -1.935086965560913, + "logps/chosen": -294.13031005859375, + "logps/rejected": -214.6305694580078, + "loss": 1564.3077, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1093236431479454, + "rewards/margins": 0.17952266335487366, + "rewards/rejected": -0.28884631395339966, + "step": 4360 + }, + { + "epoch": 0.29, + "learning_rate": 4.491911520085532e-06, + "logits/chosen": -1.9718379974365234, + "logits/rejected": -1.875333547592163, + "logps/chosen": -212.18783569335938, + "logps/rejected": -235.5560302734375, + "loss": 1678.2053, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.11780087649822235, + "rewards/margins": 0.12595582008361816, + "rewards/rejected": -0.24375669658184052, + "step": 4370 + }, + { + "epoch": 0.29, + "learning_rate": 4.488455887146075e-06, + "logits/chosen": -2.093561887741089, + "logits/rejected": -2.0697121620178223, + "logps/chosen": -184.47760009765625, + "logps/rejected": -218.77206420898438, + "loss": 1343.2975, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1436549723148346, + "rewards/margins": 0.21130797266960144, + "rewards/rejected": -0.35496294498443604, + "step": 4380 + }, + { + "epoch": 0.29, + "learning_rate": 4.484989881446654e-06, + "logits/chosen": -2.343677520751953, + "logits/rejected": -2.1531260013580322, + "logps/chosen": -215.5881805419922, + "logps/rejected": -207.2852325439453, + "loss": 2147.8211, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1311216652393341, + "rewards/margins": 0.09641782194375992, + "rewards/rejected": -0.22753946483135223, + "step": 4390 + }, + { + "epoch": 0.29, + "learning_rate": 4.481513521067654e-06, + "logits/chosen": -2.3038599491119385, + "logits/rejected": -1.9719699621200562, + "logps/chosen": -241.0018768310547, + "logps/rejected": -224.58663940429688, + "loss": 1718.8639, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16227325797080994, + "rewards/margins": 0.1900053471326828, + "rewards/rejected": -0.35227862000465393, + "step": 4400 + }, + { + "epoch": 0.29, + "eval_logits/chosen": -2.201906681060791, + "eval_logits/rejected": -2.0237557888031006, + "eval_logps/chosen": -250.08668518066406, + "eval_logps/rejected": -245.08804321289062, + "eval_loss": 1853.157470703125, + "eval_rewards/accuracies": 0.6579999923706055, + "eval_rewards/chosen": -0.18081730604171753, + "eval_rewards/margins": 0.15394414961338043, + "eval_rewards/rejected": -0.33476147055625916, + "eval_runtime": 723.7701, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.382, + "step": 4400 + }, + { + "epoch": 0.29, + "learning_rate": 4.478026824143473e-06, + "logits/chosen": -2.2210612297058105, + "logits/rejected": -2.091966152191162, + "logps/chosen": -287.1651306152344, + "logps/rejected": -248.45028686523438, + "loss": 1764.9635, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.21103541553020477, + "rewards/margins": 0.17852582037448883, + "rewards/rejected": -0.3895612359046936, + "step": 4410 + }, + { + "epoch": 0.29, + "learning_rate": 4.474529808862429e-06, + "logits/chosen": -2.106393814086914, + "logits/rejected": -2.0460286140441895, + "logps/chosen": -207.8838653564453, + "logps/rejected": -240.6083984375, + "loss": 1983.9123, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1906355768442154, + "rewards/margins": 0.15100248157978058, + "rewards/rejected": -0.3416380286216736, + "step": 4420 + }, + { + "epoch": 0.29, + "learning_rate": 4.471022493466669e-06, + "logits/chosen": -2.2233099937438965, + "logits/rejected": -1.9008467197418213, + "logps/chosen": -322.4415588378906, + "logps/rejected": -255.49288940429688, + "loss": 1897.2184, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2050246298313141, + "rewards/margins": 0.10062649101018906, + "rewards/rejected": -0.30565109848976135, + "step": 4430 + }, + { + "epoch": 0.29, + "learning_rate": 4.467504896252066e-06, + "logits/chosen": -2.2593376636505127, + "logits/rejected": -2.158722400665283, + "logps/chosen": -271.5922546386719, + "logps/rejected": -254.9861602783203, + "loss": 1963.3658, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.21890942752361298, + "rewards/margins": 0.1213611364364624, + "rewards/rejected": -0.3402705788612366, + "step": 4440 + }, + { + "epoch": 0.29, + "learning_rate": 4.463977035568132e-06, + "logits/chosen": -2.1187386512756348, + "logits/rejected": -2.3420820236206055, + "logps/chosen": -232.56396484375, + "logps/rejected": -291.1766052246094, + "loss": 2271.5484, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.20181193947792053, + "rewards/margins": 0.0690707340836525, + "rewards/rejected": -0.27088266611099243, + "step": 4450 + }, + { + "epoch": 0.29, + "learning_rate": 4.460438929817914e-06, + "logits/chosen": -2.1885595321655273, + "logits/rejected": -2.0391077995300293, + "logps/chosen": -223.7105712890625, + "logps/rejected": -232.64151000976562, + "loss": 1891.498, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.17290803790092468, + "rewards/margins": 0.13219548761844635, + "rewards/rejected": -0.30510348081588745, + "step": 4460 + }, + { + "epoch": 0.29, + "learning_rate": 4.456890597457907e-06, + "logits/chosen": -2.036794662475586, + "logits/rejected": -2.0769336223602295, + "logps/chosen": -233.7548370361328, + "logps/rejected": -265.60992431640625, + "loss": 1752.2451, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21619176864624023, + "rewards/margins": 0.13650615513324738, + "rewards/rejected": -0.3526979088783264, + "step": 4470 + }, + { + "epoch": 0.29, + "learning_rate": 4.453332056997951e-06, + "logits/chosen": -2.153390407562256, + "logits/rejected": -2.204169750213623, + "logps/chosen": -192.40567016601562, + "logps/rejected": -205.3812713623047, + "loss": 1493.2564, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12446187436580658, + "rewards/margins": 0.1730065643787384, + "rewards/rejected": -0.2974683940410614, + "step": 4480 + }, + { + "epoch": 0.29, + "learning_rate": 4.449763327001134e-06, + "logits/chosen": -2.1880500316619873, + "logits/rejected": -2.103924512863159, + "logps/chosen": -206.2200164794922, + "logps/rejected": -247.2500457763672, + "loss": 1935.248, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1745779663324356, + "rewards/margins": 0.12614604830741882, + "rewards/rejected": -0.3007240295410156, + "step": 4490 + }, + { + "epoch": 0.29, + "learning_rate": 4.446184426083702e-06, + "logits/chosen": -2.176182985305786, + "logits/rejected": -1.961561918258667, + "logps/chosen": -213.0888214111328, + "logps/rejected": -240.0130615234375, + "loss": 1744.0963, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.19524452090263367, + "rewards/margins": 0.18556657433509827, + "rewards/rejected": -0.3808111250400543, + "step": 4500 + }, + { + "epoch": 0.29, + "eval_logits/chosen": -2.2259559631347656, + "eval_logits/rejected": -2.0475778579711914, + "eval_logps/chosen": -252.73110961914062, + "eval_logps/rejected": -246.5689239501953, + "eval_loss": 1841.0155029296875, + "eval_rewards/accuracies": 0.6554999947547913, + "eval_rewards/chosen": -0.2072615772485733, + "eval_rewards/margins": 0.14230865240097046, + "eval_rewards/rejected": -0.3495701849460602, + "eval_runtime": 723.0161, + "eval_samples_per_second": 2.766, + "eval_steps_per_second": 1.383, + "step": 4500 + }, + { + "epoch": 0.3, + "learning_rate": 4.442595372914954e-06, + "logits/chosen": -2.2738208770751953, + "logits/rejected": -2.0263562202453613, + "logps/chosen": -253.7425994873047, + "logps/rejected": -182.83026123046875, + "loss": 1683.9623, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1676226258277893, + "rewards/margins": 0.14361922442913055, + "rewards/rejected": -0.3112418055534363, + "step": 4510 + }, + { + "epoch": 0.3, + "learning_rate": 4.43899618621715e-06, + "logits/chosen": -2.2293381690979004, + "logits/rejected": -2.0211691856384277, + "logps/chosen": -275.49951171875, + "logps/rejected": -293.45526123046875, + "loss": 2163.3051, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.24445602297782898, + "rewards/margins": 0.1475904881954193, + "rewards/rejected": -0.3920465111732483, + "step": 4520 + }, + { + "epoch": 0.3, + "learning_rate": 4.4353868847654105e-06, + "logits/chosen": -2.3481974601745605, + "logits/rejected": -2.113680601119995, + "logps/chosen": -262.9469909667969, + "logps/rejected": -245.699462890625, + "loss": 1954.6063, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17088761925697327, + "rewards/margins": 0.12820035219192505, + "rewards/rejected": -0.2990879416465759, + "step": 4530 + }, + { + "epoch": 0.3, + "learning_rate": 4.43176748738762e-06, + "logits/chosen": -2.254246711730957, + "logits/rejected": -2.0310864448547363, + "logps/chosen": -250.73941040039062, + "logps/rejected": -270.41595458984375, + "loss": 1790.5406, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.18791458010673523, + "rewards/margins": 0.15288159251213074, + "rewards/rejected": -0.34079617261886597, + "step": 4540 + }, + { + "epoch": 0.3, + "learning_rate": 4.4281380129643295e-06, + "logits/chosen": -2.1591944694519043, + "logits/rejected": -1.999447226524353, + "logps/chosen": -241.9261016845703, + "logps/rejected": -247.62173461914062, + "loss": 1913.1277, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12175941467285156, + "rewards/margins": 0.16169685125350952, + "rewards/rejected": -0.2834562659263611, + "step": 4550 + }, + { + "epoch": 0.3, + "learning_rate": 4.424498480428654e-06, + "logits/chosen": -2.1831679344177246, + "logits/rejected": -2.078092098236084, + "logps/chosen": -261.2657775878906, + "logps/rejected": -229.1588897705078, + "loss": 2331.508, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.13993816077709198, + "rewards/margins": 0.0791858583688736, + "rewards/rejected": -0.21912403404712677, + "step": 4560 + }, + { + "epoch": 0.3, + "learning_rate": 4.420848908766178e-06, + "logits/chosen": -2.304107189178467, + "logits/rejected": -2.232893943786621, + "logps/chosen": -217.4763946533203, + "logps/rejected": -234.603759765625, + "loss": 2148.2904, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.10249540954828262, + "rewards/margins": 0.08247154206037521, + "rewards/rejected": -0.18496695160865784, + "step": 4570 + }, + { + "epoch": 0.3, + "learning_rate": 4.417189317014855e-06, + "logits/chosen": -2.1350772380828857, + "logits/rejected": -2.364804983139038, + "logps/chosen": -214.90951538085938, + "logps/rejected": -251.42098999023438, + "loss": 2562.5131, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.14113643765449524, + "rewards/margins": 0.058145761489868164, + "rewards/rejected": -0.1992821991443634, + "step": 4580 + }, + { + "epoch": 0.3, + "learning_rate": 4.41351972426491e-06, + "logits/chosen": -2.0422589778900146, + "logits/rejected": -2.074331760406494, + "logps/chosen": -263.0894470214844, + "logps/rejected": -328.100830078125, + "loss": 1863.9609, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1635008156299591, + "rewards/margins": 0.10958373546600342, + "rewards/rejected": -0.2730845510959625, + "step": 4590 + }, + { + "epoch": 0.3, + "learning_rate": 4.409840149658735e-06, + "logits/chosen": -2.1350038051605225, + "logits/rejected": -1.894366979598999, + "logps/chosen": -295.7662353515625, + "logps/rejected": -260.21661376953125, + "loss": 1741.5717, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11248985677957535, + "rewards/margins": 0.14602139592170715, + "rewards/rejected": -0.2585112452507019, + "step": 4600 + }, + { + "epoch": 0.3, + "eval_logits/chosen": -2.2153773307800293, + "eval_logits/rejected": -2.0375046730041504, + "eval_logps/chosen": -245.6457061767578, + "eval_logps/rejected": -236.9356231689453, + "eval_loss": 1874.8072509765625, + "eval_rewards/accuracies": 0.6439999938011169, + "eval_rewards/chosen": -0.13640740513801575, + "eval_rewards/margins": 0.11682987213134766, + "eval_rewards/rejected": -0.2532372772693634, + "eval_runtime": 726.1194, + "eval_samples_per_second": 2.754, + "eval_steps_per_second": 1.377, + "step": 4600 + }, + { + "epoch": 0.3, + "learning_rate": 4.4061506123907925e-06, + "logits/chosen": -2.129925489425659, + "logits/rejected": -1.9715219736099243, + "logps/chosen": -276.8015441894531, + "logps/rejected": -247.16293334960938, + "loss": 1926.1814, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12709729373455048, + "rewards/margins": 0.1066756621003151, + "rewards/rejected": -0.23377294838428497, + "step": 4610 + }, + { + "epoch": 0.3, + "learning_rate": 4.402451131707519e-06, + "logits/chosen": -2.346142530441284, + "logits/rejected": -1.9041502475738525, + "logps/chosen": -224.49453735351562, + "logps/rejected": -164.30020141601562, + "loss": 1624.9742, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1502912938594818, + "rewards/margins": 0.1528705507516861, + "rewards/rejected": -0.3031618595123291, + "step": 4620 + }, + { + "epoch": 0.3, + "learning_rate": 4.398741726907215e-06, + "logits/chosen": -2.377230167388916, + "logits/rejected": -2.0662665367126465, + "logps/chosen": -294.00048828125, + "logps/rejected": -265.50164794921875, + "loss": 1766.6145, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14743125438690186, + "rewards/margins": 0.11827798932790756, + "rewards/rejected": -0.2657092213630676, + "step": 4630 + }, + { + "epoch": 0.3, + "learning_rate": 4.395022417339955e-06, + "logits/chosen": -2.127654552459717, + "logits/rejected": -2.1667144298553467, + "logps/chosen": -225.1376495361328, + "logps/rejected": -247.256103515625, + "loss": 2098.4934, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.18512628972530365, + "rewards/margins": 0.13629302382469177, + "rewards/rejected": -0.32141929864883423, + "step": 4640 + }, + { + "epoch": 0.3, + "learning_rate": 4.391293222407479e-06, + "logits/chosen": -2.220669746398926, + "logits/rejected": -2.22794771194458, + "logps/chosen": -148.03746032714844, + "logps/rejected": -177.59378051757812, + "loss": 1985.2336, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.11480303108692169, + "rewards/margins": 0.1132628470659256, + "rewards/rejected": -0.22806589305400848, + "step": 4650 + }, + { + "epoch": 0.3, + "learning_rate": 4.387554161563094e-06, + "logits/chosen": -2.2385716438293457, + "logits/rejected": -2.1579957008361816, + "logps/chosen": -218.38253784179688, + "logps/rejected": -224.18582153320312, + "loss": 1701.8543, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.19518354535102844, + "rewards/margins": 0.17622928321361542, + "rewards/rejected": -0.37141281366348267, + "step": 4660 + }, + { + "epoch": 0.31, + "learning_rate": 4.383805254311575e-06, + "logits/chosen": -2.4300432205200195, + "logits/rejected": -2.0662426948547363, + "logps/chosen": -271.3805236816406, + "logps/rejected": -238.52798461914062, + "loss": 1771.1676, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15239445865154266, + "rewards/margins": 0.1271374523639679, + "rewards/rejected": -0.27953192591667175, + "step": 4670 + }, + { + "epoch": 0.31, + "learning_rate": 4.380046520209056e-06, + "logits/chosen": -2.286498546600342, + "logits/rejected": -1.929011583328247, + "logps/chosen": -217.7688751220703, + "logps/rejected": -208.69705200195312, + "loss": 1793.9574, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.17502537369728088, + "rewards/margins": 0.14767295122146606, + "rewards/rejected": -0.32269835472106934, + "step": 4680 + }, + { + "epoch": 0.31, + "learning_rate": 4.376277978862936e-06, + "logits/chosen": -2.1442456245422363, + "logits/rejected": -1.869377851486206, + "logps/chosen": -238.1748809814453, + "logps/rejected": -209.87319946289062, + "loss": 1805.5377, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.12186243385076523, + "rewards/margins": 0.1222982183098793, + "rewards/rejected": -0.24416062235832214, + "step": 4690 + }, + { + "epoch": 0.31, + "learning_rate": 4.372499649931774e-06, + "logits/chosen": -2.115865707397461, + "logits/rejected": -2.2537665367126465, + "logps/chosen": -226.94723510742188, + "logps/rejected": -257.67767333984375, + "loss": 1377.8717, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.17855672538280487, + "rewards/margins": 0.22774052619934082, + "rewards/rejected": -0.4062972962856293, + "step": 4700 + }, + { + "epoch": 0.31, + "eval_logits/chosen": -2.2539665699005127, + "eval_logits/rejected": -2.073241710662842, + "eval_logps/chosen": -246.1263427734375, + "eval_logps/rejected": -241.43572998046875, + "eval_loss": 1864.4312744140625, + "eval_rewards/accuracies": 0.6499999761581421, + "eval_rewards/chosen": -0.14121387898921967, + "eval_rewards/margins": 0.15702448785305023, + "eval_rewards/rejected": -0.2982383668422699, + "eval_runtime": 725.9924, + "eval_samples_per_second": 2.755, + "eval_steps_per_second": 1.377, + "step": 4700 + }, + { + "epoch": 0.31, + "learning_rate": 4.368711553125185e-06, + "logits/chosen": -2.43339204788208, + "logits/rejected": -2.213193416595459, + "logps/chosen": -289.5559997558594, + "logps/rejected": -244.2785186767578, + "loss": 1851.9451, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12365945428609848, + "rewards/margins": 0.13165977597236633, + "rewards/rejected": -0.2553192377090454, + "step": 4710 + }, + { + "epoch": 0.31, + "learning_rate": 4.364913708203734e-06, + "logits/chosen": -2.3437886238098145, + "logits/rejected": -2.010408878326416, + "logps/chosen": -294.05780029296875, + "logps/rejected": -237.4075927734375, + "loss": 1524.1236, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.0916152149438858, + "rewards/margins": 0.18767061829566956, + "rewards/rejected": -0.27928584814071655, + "step": 4720 + }, + { + "epoch": 0.31, + "learning_rate": 4.361106134978844e-06, + "logits/chosen": -2.2243950366973877, + "logits/rejected": -2.024705648422241, + "logps/chosen": -278.34991455078125, + "logps/rejected": -280.64630126953125, + "loss": 1785.0814, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07919471710920334, + "rewards/margins": 0.1524711400270462, + "rewards/rejected": -0.23166581988334656, + "step": 4730 + }, + { + "epoch": 0.31, + "learning_rate": 4.357288853312681e-06, + "logits/chosen": -2.296668767929077, + "logits/rejected": -2.223452568054199, + "logps/chosen": -292.02313232421875, + "logps/rejected": -300.81060791015625, + "loss": 1704.7045, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.08545318245887756, + "rewards/margins": 0.13524378836154938, + "rewards/rejected": -0.22069695591926575, + "step": 4740 + }, + { + "epoch": 0.31, + "learning_rate": 4.353461883118056e-06, + "logits/chosen": -2.211477279663086, + "logits/rejected": -2.0765414237976074, + "logps/chosen": -243.3046875, + "logps/rejected": -231.37710571289062, + "loss": 2130.0371, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.140141099691391, + "rewards/margins": 0.09223539382219315, + "rewards/rejected": -0.23237650096416473, + "step": 4750 + }, + { + "epoch": 0.31, + "learning_rate": 4.34962524435832e-06, + "logits/chosen": -2.0958666801452637, + "logits/rejected": -1.9986375570297241, + "logps/chosen": -227.4104461669922, + "logps/rejected": -209.27566528320312, + "loss": 1820.6793, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07730014622211456, + "rewards/margins": 0.18698593974113464, + "rewards/rejected": -0.264286071062088, + "step": 4760 + }, + { + "epoch": 0.31, + "learning_rate": 4.34577895704726e-06, + "logits/chosen": -2.3277480602264404, + "logits/rejected": -2.160508155822754, + "logps/chosen": -270.6835021972656, + "logps/rejected": -262.9879150390625, + "loss": 1982.2266, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09069456160068512, + "rewards/margins": 0.18028482794761658, + "rewards/rejected": -0.2709794044494629, + "step": 4770 + }, + { + "epoch": 0.31, + "learning_rate": 4.3419230412489954e-06, + "logits/chosen": -2.4157934188842773, + "logits/rejected": -2.1890101432800293, + "logps/chosen": -299.4608459472656, + "logps/rejected": -239.5835418701172, + "loss": 1680.4857, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.10093015432357788, + "rewards/margins": 0.1618657112121582, + "rewards/rejected": -0.2627958655357361, + "step": 4780 + }, + { + "epoch": 0.31, + "learning_rate": 4.338057517077872e-06, + "logits/chosen": -2.3592638969421387, + "logits/rejected": -1.976302146911621, + "logps/chosen": -200.95852661132812, + "logps/rejected": -185.67758178710938, + "loss": 1029.8841, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06917314231395721, + "rewards/margins": 0.28071340918540955, + "rewards/rejected": -0.34988656640052795, + "step": 4790 + }, + { + "epoch": 0.31, + "learning_rate": 4.334182404698356e-06, + "logits/chosen": -2.3497655391693115, + "logits/rejected": -1.9321916103363037, + "logps/chosen": -242.8307647705078, + "logps/rejected": -180.7891387939453, + "loss": 1664.0443, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12723731994628906, + "rewards/margins": 0.16358184814453125, + "rewards/rejected": -0.2908191680908203, + "step": 4800 + }, + { + "epoch": 0.31, + "eval_logits/chosen": -2.2518417835235596, + "eval_logits/rejected": -2.0734591484069824, + "eval_logps/chosen": -242.2026824951172, + "eval_logps/rejected": -236.78143310546875, + "eval_loss": 1858.1903076171875, + "eval_rewards/accuracies": 0.6424999833106995, + "eval_rewards/chosen": -0.10197733342647552, + "eval_rewards/margins": 0.14971810579299927, + "eval_rewards/rejected": -0.2516954243183136, + "eval_runtime": 724.1682, + "eval_samples_per_second": 2.762, + "eval_steps_per_second": 1.381, + "step": 4800 + }, + { + "epoch": 0.31, + "learning_rate": 4.330297724324933e-06, + "logits/chosen": -2.511172294616699, + "logits/rejected": -1.9903972148895264, + "logps/chosen": -316.10528564453125, + "logps/rejected": -226.95077514648438, + "loss": 1947.774, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07693891227245331, + "rewards/margins": 0.12045726925134659, + "rewards/rejected": -0.1973962038755417, + "step": 4810 + }, + { + "epoch": 0.32, + "learning_rate": 4.326403496221999e-06, + "logits/chosen": -2.19785475730896, + "logits/rejected": -2.108128547668457, + "logps/chosen": -170.43618774414062, + "logps/rejected": -152.4991455078125, + "loss": 2344.3492, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.09007158875465393, + "rewards/margins": 0.08842898905277252, + "rewards/rejected": -0.17850057780742645, + "step": 4820 + }, + { + "epoch": 0.32, + "learning_rate": 4.322499740703755e-06, + "logits/chosen": -2.1665303707122803, + "logits/rejected": -2.2699267864227295, + "logps/chosen": -202.8117218017578, + "logps/rejected": -240.5999298095703, + "loss": 1846.5477, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.10712514072656631, + "rewards/margins": 0.13264909386634827, + "rewards/rejected": -0.23977422714233398, + "step": 4830 + }, + { + "epoch": 0.32, + "learning_rate": 4.318586478134101e-06, + "logits/chosen": -2.194655179977417, + "logits/rejected": -2.1641483306884766, + "logps/chosen": -201.90403747558594, + "logps/rejected": -171.35284423828125, + "loss": 2246.6045, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.09236544370651245, + "rewards/margins": 0.09969266504049301, + "rewards/rejected": -0.19205810129642487, + "step": 4840 + }, + { + "epoch": 0.32, + "learning_rate": 4.314663728926534e-06, + "logits/chosen": -2.432751178741455, + "logits/rejected": -2.1915640830993652, + "logps/chosen": -269.34002685546875, + "logps/rejected": -270.07049560546875, + "loss": 1822.3117, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1246429905295372, + "rewards/margins": 0.11308778822422028, + "rewards/rejected": -0.23773078620433807, + "step": 4850 + }, + { + "epoch": 0.32, + "learning_rate": 4.310731513544033e-06, + "logits/chosen": -2.244410753250122, + "logits/rejected": -2.081212282180786, + "logps/chosen": -255.2774658203125, + "logps/rejected": -223.5243682861328, + "loss": 1767.9674, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12419673055410385, + "rewards/margins": 0.14326511323451996, + "rewards/rejected": -0.2674618065357208, + "step": 4860 + }, + { + "epoch": 0.32, + "learning_rate": 4.30678985249896e-06, + "logits/chosen": -2.251373767852783, + "logits/rejected": -2.190117835998535, + "logps/chosen": -170.6262664794922, + "logps/rejected": -206.30154418945312, + "loss": 1493.1467, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.13476242125034332, + "rewards/margins": 0.1741175353527069, + "rewards/rejected": -0.3088799715042114, + "step": 4870 + }, + { + "epoch": 0.32, + "learning_rate": 4.302838766352952e-06, + "logits/chosen": -2.2527387142181396, + "logits/rejected": -2.0389404296875, + "logps/chosen": -271.576904296875, + "logps/rejected": -252.8636932373047, + "loss": 1568.2116, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15983930230140686, + "rewards/margins": 0.17078331112861633, + "rewards/rejected": -0.3306226134300232, + "step": 4880 + }, + { + "epoch": 0.32, + "learning_rate": 4.298878275716806e-06, + "logits/chosen": -2.1781983375549316, + "logits/rejected": -2.1654231548309326, + "logps/chosen": -212.4600067138672, + "logps/rejected": -224.9856414794922, + "loss": 2034.7523, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.20580561459064484, + "rewards/margins": 0.15322616696357727, + "rewards/rejected": -0.3590317964553833, + "step": 4890 + }, + { + "epoch": 0.32, + "learning_rate": 4.294908401250386e-06, + "logits/chosen": -2.4028632640838623, + "logits/rejected": -1.969160795211792, + "logps/chosen": -232.218994140625, + "logps/rejected": -203.0339813232422, + "loss": 1791.327, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.17475949227809906, + "rewards/margins": 0.15347324311733246, + "rewards/rejected": -0.3282327353954315, + "step": 4900 + }, + { + "epoch": 0.32, + "eval_logits/chosen": -2.2827320098876953, + "eval_logits/rejected": -2.102172374725342, + "eval_logps/chosen": -249.71177673339844, + "eval_logps/rejected": -244.05223083496094, + "eval_loss": 1849.7412109375, + "eval_rewards/accuracies": 0.6455000042915344, + "eval_rewards/chosen": -0.1770685464143753, + "eval_rewards/margins": 0.14733484387397766, + "eval_rewards/rejected": -0.3244033753871918, + "eval_runtime": 723.0729, + "eval_samples_per_second": 2.766, + "eval_steps_per_second": 1.383, + "step": 4900 + }, + { + "epoch": 0.32, + "learning_rate": 4.290929163662498e-06, + "logits/chosen": -2.1477413177490234, + "logits/rejected": -1.9333131313323975, + "logps/chosen": -285.38873291015625, + "logps/rejected": -240.5457763671875, + "loss": 2015.675, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.15653455257415771, + "rewards/margins": 0.12873533368110657, + "rewards/rejected": -0.2852698862552643, + "step": 4910 + }, + { + "epoch": 0.32, + "learning_rate": 4.286940583710796e-06, + "logits/chosen": -2.3153796195983887, + "logits/rejected": -2.2042593955993652, + "logps/chosen": -308.6776123046875, + "logps/rejected": -272.7545471191406, + "loss": 1321.0888, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.15928933024406433, + "rewards/margins": 0.20706868171691895, + "rewards/rejected": -0.3663580119609833, + "step": 4920 + }, + { + "epoch": 0.32, + "learning_rate": 4.282942682201667e-06, + "logits/chosen": -2.186399221420288, + "logits/rejected": -1.9242067337036133, + "logps/chosen": -267.5376281738281, + "logps/rejected": -242.9633331298828, + "loss": 1921.7139, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15429851412773132, + "rewards/margins": 0.15012209117412567, + "rewards/rejected": -0.3044206202030182, + "step": 4930 + }, + { + "epoch": 0.32, + "learning_rate": 4.278935479990123e-06, + "logits/chosen": -2.474672555923462, + "logits/rejected": -2.2394700050354004, + "logps/chosen": -218.3623046875, + "logps/rejected": -187.2513427734375, + "loss": 1960.6902, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.13698963820934296, + "rewards/margins": 0.11860468238592148, + "rewards/rejected": -0.25559431314468384, + "step": 4940 + }, + { + "epoch": 0.32, + "learning_rate": 4.274918997979695e-06, + "logits/chosen": -2.2295145988464355, + "logits/rejected": -2.2632548809051514, + "logps/chosen": -211.71249389648438, + "logps/rejected": -226.74258422851562, + "loss": 2047.2387, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.18077467381954193, + "rewards/margins": 0.1329469382762909, + "rewards/rejected": -0.31372159719467163, + "step": 4950 + }, + { + "epoch": 0.32, + "learning_rate": 4.270893257122319e-06, + "logits/chosen": -2.149606227874756, + "logits/rejected": -1.9314075708389282, + "logps/chosen": -247.7263641357422, + "logps/rejected": -294.9955749511719, + "loss": 1735.017, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.19004389643669128, + "rewards/margins": 0.17759864032268524, + "rewards/rejected": -0.36764252185821533, + "step": 4960 + }, + { + "epoch": 0.33, + "learning_rate": 4.266858278418232e-06, + "logits/chosen": -2.164851427078247, + "logits/rejected": -1.9139404296875, + "logps/chosen": -253.0793914794922, + "logps/rejected": -243.8963623046875, + "loss": 2009.3545, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1647520512342453, + "rewards/margins": 0.11168110370635986, + "rewards/rejected": -0.27643316984176636, + "step": 4970 + }, + { + "epoch": 0.33, + "learning_rate": 4.26281408291586e-06, + "logits/chosen": -2.3956422805786133, + "logits/rejected": -2.10038685798645, + "logps/chosen": -259.8249206542969, + "logps/rejected": -248.13607788085938, + "loss": 1627.1444, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17487621307373047, + "rewards/margins": 0.1784350723028183, + "rewards/rejected": -0.3533112704753876, + "step": 4980 + }, + { + "epoch": 0.33, + "learning_rate": 4.258760691711706e-06, + "logits/chosen": -2.3154006004333496, + "logits/rejected": -2.1934704780578613, + "logps/chosen": -216.1031494140625, + "logps/rejected": -224.42495727539062, + "loss": 1871.7553, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.18641582131385803, + "rewards/margins": 0.12914636731147766, + "rewards/rejected": -0.3155621886253357, + "step": 4990 + }, + { + "epoch": 0.33, + "learning_rate": 4.254698125950247e-06, + "logits/chosen": -2.5614352226257324, + "logits/rejected": -2.3224945068359375, + "logps/chosen": -312.9407958984375, + "logps/rejected": -283.60455322265625, + "loss": 1540.4815, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11926261335611343, + "rewards/margins": 0.17329421639442444, + "rewards/rejected": -0.2925568222999573, + "step": 5000 + }, + { + "epoch": 0.33, + "eval_logits/chosen": -2.3258872032165527, + "eval_logits/rejected": -2.142472743988037, + "eval_logps/chosen": -249.7621307373047, + "eval_logps/rejected": -244.18833923339844, + "eval_loss": 1836.64892578125, + "eval_rewards/accuracies": 0.6495000123977661, + "eval_rewards/chosen": -0.17757174372673035, + "eval_rewards/margins": 0.1481926590204239, + "eval_rewards/rejected": -0.32576441764831543, + "eval_runtime": 723.2528, + "eval_samples_per_second": 2.765, + "eval_steps_per_second": 1.383, + "step": 5000 + }, + { + "epoch": 0.33, + "learning_rate": 4.250626406823815e-06, + "logits/chosen": -2.3754916191101074, + "logits/rejected": -2.1217591762542725, + "logps/chosen": -237.7047576904297, + "logps/rejected": -273.30865478515625, + "loss": 1869.5283, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.20842047035694122, + "rewards/margins": 0.19425836205482483, + "rewards/rejected": -0.40267881751060486, + "step": 5010 + }, + { + "epoch": 0.33, + "learning_rate": 4.246545555572489e-06, + "logits/chosen": -2.3027451038360596, + "logits/rejected": -2.1745707988739014, + "logps/chosen": -171.08348083496094, + "logps/rejected": -212.65609741210938, + "loss": 1589.6315, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1688733845949173, + "rewards/margins": 0.16620567440986633, + "rewards/rejected": -0.33507904410362244, + "step": 5020 + }, + { + "epoch": 0.33, + "learning_rate": 4.242455593483992e-06, + "logits/chosen": -2.385986566543579, + "logits/rejected": -2.1910018920898438, + "logps/chosen": -240.56057739257812, + "logps/rejected": -203.0875701904297, + "loss": 1833.0092, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.21406681835651398, + "rewards/margins": 0.12236443907022476, + "rewards/rejected": -0.33643123507499695, + "step": 5030 + }, + { + "epoch": 0.33, + "learning_rate": 4.238356541893567e-06, + "logits/chosen": -2.2863965034484863, + "logits/rejected": -2.148052453994751, + "logps/chosen": -211.9119110107422, + "logps/rejected": -210.85281372070312, + "loss": 1867.3238, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.21778789162635803, + "rewards/margins": 0.14483478665351868, + "rewards/rejected": -0.36262261867523193, + "step": 5040 + }, + { + "epoch": 0.33, + "learning_rate": 4.234248422183876e-06, + "logits/chosen": -2.125394582748413, + "logits/rejected": -2.3238110542297363, + "logps/chosen": -255.83609008789062, + "logps/rejected": -266.41595458984375, + "loss": 1845.3883, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12901371717453003, + "rewards/margins": 0.15136227011680603, + "rewards/rejected": -0.28037601709365845, + "step": 5050 + }, + { + "epoch": 0.33, + "learning_rate": 4.230131255784884e-06, + "logits/chosen": -2.5306344032287598, + "logits/rejected": -2.224951982498169, + "logps/chosen": -273.32659912109375, + "logps/rejected": -273.3016052246094, + "loss": 1727.2801, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.15443465113639832, + "rewards/margins": 0.1877146065235138, + "rewards/rejected": -0.3421493172645569, + "step": 5060 + }, + { + "epoch": 0.33, + "learning_rate": 4.226005064173748e-06, + "logits/chosen": -2.3271052837371826, + "logits/rejected": -2.1794817447662354, + "logps/chosen": -278.8447570800781, + "logps/rejected": -310.39959716796875, + "loss": 1935.8678, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1420050710439682, + "rewards/margins": 0.1309777796268463, + "rewards/rejected": -0.2729828655719757, + "step": 5070 + }, + { + "epoch": 0.33, + "learning_rate": 4.2218698688747035e-06, + "logits/chosen": -2.1697347164154053, + "logits/rejected": -2.021205425262451, + "logps/chosen": -258.52752685546875, + "logps/rejected": -226.1105499267578, + "loss": 1880.8791, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2581823170185089, + "rewards/margins": 0.1271318644285202, + "rewards/rejected": -0.3853141665458679, + "step": 5080 + }, + { + "epoch": 0.33, + "learning_rate": 4.217725691458957e-06, + "logits/chosen": -2.477919101715088, + "logits/rejected": -2.3069875240325928, + "logps/chosen": -205.07302856445312, + "logps/rejected": -246.52072143554688, + "loss": 1650.2766, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2084888517856598, + "rewards/margins": 0.14501745998859406, + "rewards/rejected": -0.35350629687309265, + "step": 5090 + }, + { + "epoch": 0.33, + "learning_rate": 4.213572553544565e-06, + "logits/chosen": -2.3776400089263916, + "logits/rejected": -2.1426074504852295, + "logps/chosen": -259.0758361816406, + "logps/rejected": -266.2682800292969, + "loss": 1773.1312, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.18104009330272675, + "rewards/margins": 0.14556989073753357, + "rewards/rejected": -0.3266100287437439, + "step": 5100 + }, + { + "epoch": 0.33, + "eval_logits/chosen": -2.3287124633789062, + "eval_logits/rejected": -2.1456401348114014, + "eval_logps/chosen": -246.8673858642578, + "eval_logps/rejected": -242.05673217773438, + "eval_loss": 1836.536865234375, + "eval_rewards/accuracies": 0.652999997138977, + "eval_rewards/chosen": -0.1486242264509201, + "eval_rewards/margins": 0.1558239907026291, + "eval_rewards/rejected": -0.3044482469558716, + "eval_runtime": 722.0025, + "eval_samples_per_second": 2.77, + "eval_steps_per_second": 1.385, + "step": 5100 + }, + { + "epoch": 0.33, + "learning_rate": 4.209410476796331e-06, + "logits/chosen": -2.253725290298462, + "logits/rejected": -2.199178695678711, + "logps/chosen": -186.88389587402344, + "logps/rejected": -193.34152221679688, + "loss": 1685.0238, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15792416036128998, + "rewards/margins": 0.16446875035762787, + "rewards/rejected": -0.32239291071891785, + "step": 5110 + }, + { + "epoch": 0.33, + "learning_rate": 4.205239482925686e-06, + "logits/chosen": -2.1166889667510986, + "logits/rejected": -2.1698646545410156, + "logps/chosen": -204.41046142578125, + "logps/rejected": -239.6005401611328, + "loss": 2057.5961, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16595105826854706, + "rewards/margins": 0.13063675165176392, + "rewards/rejected": -0.29658785462379456, + "step": 5120 + }, + { + "epoch": 0.34, + "learning_rate": 4.201059593690577e-06, + "logits/chosen": -2.375741720199585, + "logits/rejected": -2.28941011428833, + "logps/chosen": -242.7406005859375, + "logps/rejected": -229.38473510742188, + "loss": 1988.8654, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1715283840894699, + "rewards/margins": 0.12073174864053726, + "rewards/rejected": -0.29226014018058777, + "step": 5130 + }, + { + "epoch": 0.34, + "learning_rate": 4.196870830895354e-06, + "logits/chosen": -2.1859264373779297, + "logits/rejected": -2.2038087844848633, + "logps/chosen": -274.9366760253906, + "logps/rejected": -337.29449462890625, + "loss": 2076.1455, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.17189116775989532, + "rewards/margins": 0.12850435078144073, + "rewards/rejected": -0.30039554834365845, + "step": 5140 + }, + { + "epoch": 0.34, + "learning_rate": 4.192673216390657e-06, + "logits/chosen": -2.3719637393951416, + "logits/rejected": -2.1126458644866943, + "logps/chosen": -250.2086181640625, + "logps/rejected": -223.68789672851562, + "loss": 2109.3109, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1723567098379135, + "rewards/margins": 0.11806859821081161, + "rewards/rejected": -0.29042530059814453, + "step": 5150 + }, + { + "epoch": 0.34, + "learning_rate": 4.188466772073296e-06, + "logits/chosen": -2.469306230545044, + "logits/rejected": -2.159566879272461, + "logps/chosen": -235.36898803710938, + "logps/rejected": -225.29098510742188, + "loss": 1837.1287, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.14844447374343872, + "rewards/margins": 0.1068364828824997, + "rewards/rejected": -0.2552809417247772, + "step": 5160 + }, + { + "epoch": 0.34, + "learning_rate": 4.184251519886148e-06, + "logits/chosen": -2.2152698040008545, + "logits/rejected": -2.312568187713623, + "logps/chosen": -219.24130249023438, + "logps/rejected": -255.0888671875, + "loss": 2291.0273, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.23133759200572968, + "rewards/margins": 0.0719255581498146, + "rewards/rejected": -0.3032631576061249, + "step": 5170 + }, + { + "epoch": 0.34, + "learning_rate": 4.180027481818033e-06, + "logits/chosen": -2.32781720161438, + "logits/rejected": -2.3139772415161133, + "logps/chosen": -280.49932861328125, + "logps/rejected": -258.47967529296875, + "loss": 1658.382, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.13801197707653046, + "rewards/margins": 0.15813925862312317, + "rewards/rejected": -0.29615122079849243, + "step": 5180 + }, + { + "epoch": 0.34, + "learning_rate": 4.175794679903602e-06, + "logits/chosen": -2.3832285404205322, + "logits/rejected": -2.149759531021118, + "logps/chosen": -239.1299591064453, + "logps/rejected": -183.6186981201172, + "loss": 1757.3795, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11489125341176987, + "rewards/margins": 0.16845735907554626, + "rewards/rejected": -0.28334861993789673, + "step": 5190 + }, + { + "epoch": 0.34, + "learning_rate": 4.171553136223222e-06, + "logits/chosen": -2.3328781127929688, + "logits/rejected": -2.3383240699768066, + "logps/chosen": -283.13861083984375, + "logps/rejected": -312.5195007324219, + "loss": 1598.2573, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15817800164222717, + "rewards/margins": 0.1867099553346634, + "rewards/rejected": -0.34488797187805176, + "step": 5200 + }, + { + "epoch": 0.34, + "eval_logits/chosen": -2.352234363555908, + "eval_logits/rejected": -2.1671369075775146, + "eval_logps/chosen": -246.12818908691406, + "eval_logps/rejected": -239.1405792236328, + "eval_loss": 1844.184326171875, + "eval_rewards/accuracies": 0.6420000195503235, + "eval_rewards/chosen": -0.1412324607372284, + "eval_rewards/margins": 0.13405431807041168, + "eval_rewards/rejected": -0.2752867639064789, + "eval_runtime": 722.7781, + "eval_samples_per_second": 2.767, + "eval_steps_per_second": 1.384, + "step": 5200 + }, + { + "epoch": 0.34, + "learning_rate": 4.167302872902865e-06, + "logits/chosen": -2.3499057292938232, + "logits/rejected": -2.2576987743377686, + "logps/chosen": -263.365234375, + "logps/rejected": -274.6778869628906, + "loss": 1414.8517, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11897093057632446, + "rewards/margins": 0.2325071394443512, + "rewards/rejected": -0.35147807002067566, + "step": 5210 + }, + { + "epoch": 0.34, + "learning_rate": 4.163043912113985e-06, + "logits/chosen": -2.392256259918213, + "logits/rejected": -2.1617372035980225, + "logps/chosen": -271.14300537109375, + "logps/rejected": -249.79519653320312, + "loss": 1866.4203, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12102854251861572, + "rewards/margins": 0.12646819651126862, + "rewards/rejected": -0.24749675393104553, + "step": 5220 + }, + { + "epoch": 0.34, + "learning_rate": 4.15877627607341e-06, + "logits/chosen": -2.1816821098327637, + "logits/rejected": -2.0997190475463867, + "logps/chosen": -231.6128387451172, + "logps/rejected": -217.5960235595703, + "loss": 1985.5357, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.15018010139465332, + "rewards/margins": 0.10205508768558502, + "rewards/rejected": -0.25223520398139954, + "step": 5230 + }, + { + "epoch": 0.34, + "learning_rate": 4.154499987043217e-06, + "logits/chosen": -2.422106981277466, + "logits/rejected": -2.2051637172698975, + "logps/chosen": -237.9840545654297, + "logps/rejected": -231.46310424804688, + "loss": 1672.8949, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13780474662780762, + "rewards/margins": 0.15658816695213318, + "rewards/rejected": -0.2943929135799408, + "step": 5240 + }, + { + "epoch": 0.34, + "learning_rate": 4.150215067330625e-06, + "logits/chosen": -2.2486937046051025, + "logits/rejected": -2.2005836963653564, + "logps/chosen": -225.8989715576172, + "logps/rejected": -255.8649444580078, + "loss": 1956.0352, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.16850169003009796, + "rewards/margins": 0.1592865288257599, + "rewards/rejected": -0.32778820395469666, + "step": 5250 + }, + { + "epoch": 0.34, + "learning_rate": 4.145921539287876e-06, + "logits/chosen": -2.282233238220215, + "logits/rejected": -2.0033607482910156, + "logps/chosen": -207.8868408203125, + "logps/rejected": -204.358154296875, + "loss": 1690.5404, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.17294219136238098, + "rewards/margins": 0.1647462546825409, + "rewards/rejected": -0.3376884162425995, + "step": 5260 + }, + { + "epoch": 0.34, + "learning_rate": 4.141619425312115e-06, + "logits/chosen": -2.361851692199707, + "logits/rejected": -1.998276948928833, + "logps/chosen": -227.4818878173828, + "logps/rejected": -210.9532470703125, + "loss": 2334.2115, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.18830615282058716, + "rewards/margins": 0.07913248240947723, + "rewards/rejected": -0.2674386501312256, + "step": 5270 + }, + { + "epoch": 0.35, + "learning_rate": 4.1373087478452735e-06, + "logits/chosen": -2.4881348609924316, + "logits/rejected": -2.080901622772217, + "logps/chosen": -239.4554901123047, + "logps/rejected": -209.3617706298828, + "loss": 1104.9947, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.14214879274368286, + "rewards/margins": 0.24520206451416016, + "rewards/rejected": -0.3873508870601654, + "step": 5280 + }, + { + "epoch": 0.35, + "learning_rate": 4.132989529373959e-06, + "logits/chosen": -2.3881518840789795, + "logits/rejected": -1.9492473602294922, + "logps/chosen": -278.64263916015625, + "logps/rejected": -210.50772094726562, + "loss": 1829.0125, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2272980511188507, + "rewards/margins": 0.10863574594259262, + "rewards/rejected": -0.3359338045120239, + "step": 5290 + }, + { + "epoch": 0.35, + "learning_rate": 4.128661792429331e-06, + "logits/chosen": -2.3927273750305176, + "logits/rejected": -2.2233188152313232, + "logps/chosen": -271.4683837890625, + "logps/rejected": -287.85369873046875, + "loss": 2181.6221, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.17697831988334656, + "rewards/margins": 0.11488916724920273, + "rewards/rejected": -0.2918674945831299, + "step": 5300 + }, + { + "epoch": 0.35, + "eval_logits/chosen": -2.3462629318237305, + "eval_logits/rejected": -2.1606616973876953, + "eval_logps/chosen": -248.34812927246094, + "eval_logps/rejected": -243.24807739257812, + "eval_loss": 1859.457763671875, + "eval_rewards/accuracies": 0.6464999914169312, + "eval_rewards/chosen": -0.16343174874782562, + "eval_rewards/margins": 0.15293008089065552, + "eval_rewards/rejected": -0.31636181473731995, + "eval_runtime": 723.479, + "eval_samples_per_second": 2.764, + "eval_steps_per_second": 1.382, + "step": 5300 + }, + { + "epoch": 0.35, + "learning_rate": 4.124325559586985e-06, + "logits/chosen": -2.101710557937622, + "logits/rejected": -2.149610996246338, + "logps/chosen": -205.6929473876953, + "logps/rejected": -224.6509246826172, + "loss": 2794.7621, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.1763622760772705, + "rewards/margins": 0.06034000962972641, + "rewards/rejected": -0.23670227825641632, + "step": 5310 + }, + { + "epoch": 0.35, + "learning_rate": 4.119980853466835e-06, + "logits/chosen": -2.3114354610443115, + "logits/rejected": -1.9271361827850342, + "logps/chosen": -223.38388061523438, + "logps/rejected": -211.22024536132812, + "loss": 1752.8992, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14229492843151093, + "rewards/margins": 0.14950500428676605, + "rewards/rejected": -0.29179996252059937, + "step": 5320 + }, + { + "epoch": 0.35, + "learning_rate": 4.115627696732997e-06, + "logits/chosen": -2.236950159072876, + "logits/rejected": -2.043684720993042, + "logps/chosen": -201.57254028320312, + "logps/rejected": -193.969482421875, + "loss": 1977.1037, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.10158295929431915, + "rewards/margins": 0.11717870086431503, + "rewards/rejected": -0.21876168251037598, + "step": 5330 + }, + { + "epoch": 0.35, + "learning_rate": 4.111266112093668e-06, + "logits/chosen": -2.333730459213257, + "logits/rejected": -2.1332597732543945, + "logps/chosen": -217.0889129638672, + "logps/rejected": -250.2196502685547, + "loss": 1730.2516, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1362122893333435, + "rewards/margins": 0.1473294198513031, + "rewards/rejected": -0.283541738986969, + "step": 5340 + }, + { + "epoch": 0.35, + "learning_rate": 4.1068961223010115e-06, + "logits/chosen": -2.3114235401153564, + "logits/rejected": -2.0312106609344482, + "logps/chosen": -285.64080810546875, + "logps/rejected": -274.3442687988281, + "loss": 2088.9748, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.14940163493156433, + "rewards/margins": 0.113797627389431, + "rewards/rejected": -0.26319923996925354, + "step": 5350 + }, + { + "epoch": 0.35, + "learning_rate": 4.102517750151034e-06, + "logits/chosen": -2.374783992767334, + "logits/rejected": -2.134248971939087, + "logps/chosen": -300.9389343261719, + "logps/rejected": -243.3031768798828, + "loss": 1755.3168, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.05894102901220322, + "rewards/margins": 0.13677141070365906, + "rewards/rejected": -0.19571244716644287, + "step": 5360 + }, + { + "epoch": 0.35, + "learning_rate": 4.09813101848347e-06, + "logits/chosen": -2.248469829559326, + "logits/rejected": -2.2975306510925293, + "logps/chosen": -224.8818359375, + "logps/rejected": -256.2076416015625, + "loss": 1819.8633, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.058958910405635834, + "rewards/margins": 0.14171640574932098, + "rewards/rejected": -0.20067529380321503, + "step": 5370 + }, + { + "epoch": 0.35, + "learning_rate": 4.093735950181659e-06, + "logits/chosen": -2.213391065597534, + "logits/rejected": -2.11991548538208, + "logps/chosen": -229.80532836914062, + "logps/rejected": -271.97998046875, + "loss": 1547.5715, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.04331979155540466, + "rewards/margins": 0.17631354928016663, + "rewards/rejected": -0.2196333408355713, + "step": 5380 + }, + { + "epoch": 0.35, + "learning_rate": 4.0893325681724326e-06, + "logits/chosen": -2.3199193477630615, + "logits/rejected": -2.2667202949523926, + "logps/chosen": -269.91607666015625, + "logps/rejected": -274.984375, + "loss": 1574.0112, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11494427919387817, + "rewards/margins": 0.17347653210163116, + "rewards/rejected": -0.28842082619667053, + "step": 5390 + }, + { + "epoch": 0.35, + "learning_rate": 4.084920895425988e-06, + "logits/chosen": -2.283707618713379, + "logits/rejected": -2.2867934703826904, + "logps/chosen": -253.4472198486328, + "logps/rejected": -281.6402893066406, + "loss": 1823.2314, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16082549095153809, + "rewards/margins": 0.14548133313655853, + "rewards/rejected": -0.3063068091869354, + "step": 5400 + }, + { + "epoch": 0.35, + "eval_logits/chosen": -2.3460588455200195, + "eval_logits/rejected": -2.160550355911255, + "eval_logps/chosen": -242.38352966308594, + "eval_logps/rejected": -237.32022094726562, + "eval_loss": 1852.075439453125, + "eval_rewards/accuracies": 0.6539999842643738, + "eval_rewards/chosen": -0.10378566384315491, + "eval_rewards/margins": 0.15329769253730774, + "eval_rewards/rejected": -0.25708335638046265, + "eval_runtime": 725.1816, + "eval_samples_per_second": 2.758, + "eval_steps_per_second": 1.379, + "step": 5400 + }, + { + "epoch": 0.35, + "learning_rate": 4.080500954955769e-06, + "logits/chosen": -2.2673728466033936, + "logits/rejected": -1.979201078414917, + "logps/chosen": -267.6470642089844, + "logps/rejected": -267.2630615234375, + "loss": 1781.8809, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11775131523609161, + "rewards/margins": 0.17902211844921112, + "rewards/rejected": -0.2967734634876251, + "step": 5410 + }, + { + "epoch": 0.35, + "learning_rate": 4.076072769818354e-06, + "logits/chosen": -2.5131490230560303, + "logits/rejected": -2.0682785511016846, + "logps/chosen": -252.3065643310547, + "logps/rejected": -207.11624145507812, + "loss": 1578.0489, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.08584243804216385, + "rewards/margins": 0.16775313019752502, + "rewards/rejected": -0.2535955309867859, + "step": 5420 + }, + { + "epoch": 0.36, + "learning_rate": 4.071636363113323e-06, + "logits/chosen": -2.1224284172058105, + "logits/rejected": -2.046861171722412, + "logps/chosen": -277.6747131347656, + "logps/rejected": -232.834716796875, + "loss": 2311.8592, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.09544602036476135, + "rewards/margins": 0.09202627837657928, + "rewards/rejected": -0.18747231364250183, + "step": 5430 + }, + { + "epoch": 0.36, + "learning_rate": 4.067191757983146e-06, + "logits/chosen": -2.0987110137939453, + "logits/rejected": -2.149458885192871, + "logps/chosen": -242.80322265625, + "logps/rejected": -255.27157592773438, + "loss": 1751.2318, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1322755366563797, + "rewards/margins": 0.15682359039783478, + "rewards/rejected": -0.28909915685653687, + "step": 5440 + }, + { + "epoch": 0.36, + "learning_rate": 4.062738977613063e-06, + "logits/chosen": -2.303696870803833, + "logits/rejected": -2.177271604537964, + "logps/chosen": -244.327880859375, + "logps/rejected": -208.11434936523438, + "loss": 1977.2764, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09851580113172531, + "rewards/margins": 0.12179714441299438, + "rewards/rejected": -0.2203129231929779, + "step": 5450 + }, + { + "epoch": 0.36, + "learning_rate": 4.058278045230957e-06, + "logits/chosen": -2.2384145259857178, + "logits/rejected": -2.249328136444092, + "logps/chosen": -231.9445037841797, + "logps/rejected": -242.25210571289062, + "loss": 2202.6955, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13577644526958466, + "rewards/margins": 0.08719965070486069, + "rewards/rejected": -0.22297611832618713, + "step": 5460 + }, + { + "epoch": 0.36, + "learning_rate": 4.053808984107235e-06, + "logits/chosen": -2.4486615657806396, + "logits/rejected": -2.122954845428467, + "logps/chosen": -243.2438507080078, + "logps/rejected": -213.0865020751953, + "loss": 1996.8475, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.10375523567199707, + "rewards/margins": 0.10135851800441742, + "rewards/rejected": -0.2051137387752533, + "step": 5470 + }, + { + "epoch": 0.36, + "learning_rate": 4.04933181755471e-06, + "logits/chosen": -2.4395668506622314, + "logits/rejected": -2.3647377490997314, + "logps/chosen": -217.07693481445312, + "logps/rejected": -220.393798828125, + "loss": 1927.951, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10496320575475693, + "rewards/margins": 0.10529372841119766, + "rewards/rejected": -0.2102569341659546, + "step": 5480 + }, + { + "epoch": 0.36, + "learning_rate": 4.044846568928477e-06, + "logits/chosen": -2.3208203315734863, + "logits/rejected": -2.3979101181030273, + "logps/chosen": -272.79937744140625, + "logps/rejected": -275.45074462890625, + "loss": 1987.8191, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11275209486484528, + "rewards/margins": 0.1150924414396286, + "rewards/rejected": -0.22784452140331268, + "step": 5490 + }, + { + "epoch": 0.36, + "learning_rate": 4.040353261625788e-06, + "logits/chosen": -2.493821859359741, + "logits/rejected": -2.1161201000213623, + "logps/chosen": -285.68231201171875, + "logps/rejected": -261.2396545410156, + "loss": 1672.017, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09137077629566193, + "rewards/margins": 0.14187830686569214, + "rewards/rejected": -0.23324909806251526, + "step": 5500 + }, + { + "epoch": 0.36, + "eval_logits/chosen": -2.3531739711761475, + "eval_logits/rejected": -2.1676862239837646, + "eval_logps/chosen": -242.32733154296875, + "eval_logps/rejected": -235.50111389160156, + "eval_loss": 1837.0867919921875, + "eval_rewards/accuracies": 0.6499999761581421, + "eval_rewards/chosen": -0.10322368890047073, + "eval_rewards/margins": 0.13566839694976807, + "eval_rewards/rejected": -0.2388920933008194, + "eval_runtime": 724.875, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 5500 + }, + { + "epoch": 0.36, + "learning_rate": 4.035851919085936e-06, + "logits/chosen": -2.330836772918701, + "logits/rejected": -2.211599826812744, + "logps/chosen": -276.9023132324219, + "logps/rejected": -211.7489471435547, + "loss": 2047.8697, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09508221596479416, + "rewards/margins": 0.13558414578437805, + "rewards/rejected": -0.23066632449626923, + "step": 5510 + }, + { + "epoch": 0.36, + "learning_rate": 4.031342564790128e-06, + "logits/chosen": -2.2656824588775635, + "logits/rejected": -2.1317713260650635, + "logps/chosen": -213.4837646484375, + "logps/rejected": -224.5988311767578, + "loss": 1587.1312, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07512074708938599, + "rewards/margins": 0.15887776017189026, + "rewards/rejected": -0.23399850726127625, + "step": 5520 + }, + { + "epoch": 0.36, + "learning_rate": 4.026825222261367e-06, + "logits/chosen": -2.299889087677002, + "logits/rejected": -2.0493857860565186, + "logps/chosen": -190.53529357910156, + "logps/rejected": -191.00079345703125, + "loss": 2531.7961, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.17460234463214874, + "rewards/margins": 0.07608696818351746, + "rewards/rejected": -0.2506893277168274, + "step": 5530 + }, + { + "epoch": 0.36, + "learning_rate": 4.022299915064321e-06, + "logits/chosen": -2.3498470783233643, + "logits/rejected": -2.1532254219055176, + "logps/chosen": -312.34197998046875, + "logps/rejected": -275.0274658203125, + "loss": 1565.0791, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06494022160768509, + "rewards/margins": 0.15790972113609314, + "rewards/rejected": -0.22284996509552002, + "step": 5540 + }, + { + "epoch": 0.36, + "learning_rate": 4.017766666805213e-06, + "logits/chosen": -2.170281171798706, + "logits/rejected": -2.132956027984619, + "logps/chosen": -223.19058227539062, + "logps/rejected": -199.3626708984375, + "loss": 2156.3916, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09670194238424301, + "rewards/margins": 0.11533693969249725, + "rewards/rejected": -0.21203890442848206, + "step": 5550 + }, + { + "epoch": 0.36, + "learning_rate": 4.013225501131684e-06, + "logits/chosen": -2.372115135192871, + "logits/rejected": -2.09326171875, + "logps/chosen": -230.5290985107422, + "logps/rejected": -213.18588256835938, + "loss": 1988.3984, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.13151514530181885, + "rewards/margins": 0.09629587829113007, + "rewards/rejected": -0.22781100869178772, + "step": 5560 + }, + { + "epoch": 0.36, + "learning_rate": 4.008676441732679e-06, + "logits/chosen": -2.316739559173584, + "logits/rejected": -1.9960857629776, + "logps/chosen": -222.58773803710938, + "logps/rejected": -181.80226135253906, + "loss": 2119.3896, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.1629338562488556, + "rewards/margins": 0.07215423882007599, + "rewards/rejected": -0.23508810997009277, + "step": 5570 + }, + { + "epoch": 0.37, + "learning_rate": 4.00411951233832e-06, + "logits/chosen": -2.4594318866729736, + "logits/rejected": -2.123006582260132, + "logps/chosen": -234.63711547851562, + "logps/rejected": -204.8271942138672, + "loss": 1579.9379, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1491195410490036, + "rewards/margins": 0.14863654971122742, + "rewards/rejected": -0.2977561354637146, + "step": 5580 + }, + { + "epoch": 0.37, + "learning_rate": 3.999554736719785e-06, + "logits/chosen": -2.2024471759796143, + "logits/rejected": -2.087955951690674, + "logps/chosen": -303.3402404785156, + "logps/rejected": -271.7759704589844, + "loss": 1785.8312, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11363334953784943, + "rewards/margins": 0.11953709274530411, + "rewards/rejected": -0.23317043483257294, + "step": 5590 + }, + { + "epoch": 0.37, + "learning_rate": 3.994982138689177e-06, + "logits/chosen": -2.484562397003174, + "logits/rejected": -2.2941088676452637, + "logps/chosen": -245.7768096923828, + "logps/rejected": -256.5212097167969, + "loss": 1779.0814, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08376649767160416, + "rewards/margins": 0.12595562636852264, + "rewards/rejected": -0.2097221314907074, + "step": 5600 + }, + { + "epoch": 0.37, + "eval_logits/chosen": -2.345733880996704, + "eval_logits/rejected": -2.1606533527374268, + "eval_logps/chosen": -242.05909729003906, + "eval_logps/rejected": -235.68272399902344, + "eval_loss": 1833.4417724609375, + "eval_rewards/accuracies": 0.6480000019073486, + "eval_rewards/chosen": -0.1005413755774498, + "eval_rewards/margins": 0.14016689360141754, + "eval_rewards/rejected": -0.24070827662944794, + "eval_runtime": 723.2346, + "eval_samples_per_second": 2.765, + "eval_steps_per_second": 1.383, + "step": 5600 + }, + { + "epoch": 0.37, + "learning_rate": 3.990401742099408e-06, + "logits/chosen": -2.159879684448242, + "logits/rejected": -2.1814544200897217, + "logps/chosen": -187.76846313476562, + "logps/rejected": -188.0281524658203, + "loss": 2415.7469, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.08905371278524399, + "rewards/margins": 0.07608432322740555, + "rewards/rejected": -0.16513803601264954, + "step": 5610 + }, + { + "epoch": 0.37, + "learning_rate": 3.985813570844072e-06, + "logits/chosen": -2.278538465499878, + "logits/rejected": -2.141242027282715, + "logps/chosen": -302.8232421875, + "logps/rejected": -292.3064270019531, + "loss": 1833.266, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.0843651071190834, + "rewards/margins": 0.1703469157218933, + "rewards/rejected": -0.2547120451927185, + "step": 5620 + }, + { + "epoch": 0.37, + "learning_rate": 3.981217648857316e-06, + "logits/chosen": -2.3753318786621094, + "logits/rejected": -2.1800878047943115, + "logps/chosen": -181.85874938964844, + "logps/rejected": -199.9864959716797, + "loss": 1433.0758, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09318314492702484, + "rewards/margins": 0.16345934569835663, + "rewards/rejected": -0.25664249062538147, + "step": 5630 + }, + { + "epoch": 0.37, + "learning_rate": 3.97661400011372e-06, + "logits/chosen": -2.214317560195923, + "logits/rejected": -2.2591793537139893, + "logps/chosen": -250.345947265625, + "logps/rejected": -252.46676635742188, + "loss": 1889.515, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07318656146526337, + "rewards/margins": 0.13543672859668732, + "rewards/rejected": -0.20862329006195068, + "step": 5640 + }, + { + "epoch": 0.37, + "learning_rate": 3.972002648628174e-06, + "logits/chosen": -2.2506039142608643, + "logits/rejected": -1.9229838848114014, + "logps/chosen": -290.96160888671875, + "logps/rejected": -256.6331481933594, + "loss": 1863.8377, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.13930806517601013, + "rewards/margins": 0.13363361358642578, + "rewards/rejected": -0.2729416787624359, + "step": 5650 + }, + { + "epoch": 0.37, + "learning_rate": 3.967383618455743e-06, + "logits/chosen": -2.403590679168701, + "logits/rejected": -2.2283291816711426, + "logps/chosen": -240.9685516357422, + "logps/rejected": -274.12054443359375, + "loss": 1954.8783, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.13848820328712463, + "rewards/margins": 0.1496153622865677, + "rewards/rejected": -0.2881035804748535, + "step": 5660 + }, + { + "epoch": 0.37, + "learning_rate": 3.9627569336915515e-06, + "logits/chosen": -2.521933078765869, + "logits/rejected": -2.2157063484191895, + "logps/chosen": -255.2565460205078, + "logps/rejected": -214.8271942138672, + "loss": 1625.9281, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.07793761789798737, + "rewards/margins": 0.1624729335308075, + "rewards/rejected": -0.24041053652763367, + "step": 5670 + }, + { + "epoch": 0.37, + "learning_rate": 3.9581226184706555e-06, + "logits/chosen": -2.3785665035247803, + "logits/rejected": -2.479404926300049, + "logps/chosen": -206.0277862548828, + "logps/rejected": -290.6431884765625, + "loss": 1778.9562, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.11362370103597641, + "rewards/margins": 0.13917763531208038, + "rewards/rejected": -0.2528013586997986, + "step": 5680 + }, + { + "epoch": 0.37, + "learning_rate": 3.953480696967912e-06, + "logits/chosen": -2.0409164428710938, + "logits/rejected": -2.2784464359283447, + "logps/chosen": -216.86697387695312, + "logps/rejected": -266.86175537109375, + "loss": 1623.332, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.07495345920324326, + "rewards/margins": 0.135491281747818, + "rewards/rejected": -0.21044473350048065, + "step": 5690 + }, + { + "epoch": 0.37, + "learning_rate": 3.948831193397857e-06, + "logits/chosen": -2.2667810916900635, + "logits/rejected": -2.2484869956970215, + "logps/chosen": -177.64291381835938, + "logps/rejected": -189.92938232421875, + "loss": 2283.9326, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.11627526581287384, + "rewards/margins": 0.06460537016391754, + "rewards/rejected": -0.1808806210756302, + "step": 5700 + }, + { + "epoch": 0.37, + "eval_logits/chosen": -2.334646701812744, + "eval_logits/rejected": -2.150752544403076, + "eval_logps/chosen": -241.03411865234375, + "eval_logps/rejected": -235.60592651367188, + "eval_loss": 1836.9827880859375, + "eval_rewards/accuracies": 0.656000018119812, + "eval_rewards/chosen": -0.09029180556535721, + "eval_rewards/margins": 0.14964844286441803, + "eval_rewards/rejected": -0.23994024097919464, + "eval_runtime": 723.2472, + "eval_samples_per_second": 2.765, + "eval_steps_per_second": 1.383, + "step": 5700 + }, + { + "epoch": 0.37, + "learning_rate": 3.94417413201458e-06, + "logits/chosen": -2.228362560272217, + "logits/rejected": -2.0915615558624268, + "logps/chosen": -219.13333129882812, + "logps/rejected": -214.9150848388672, + "loss": 2166.6598, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.07561827450990677, + "rewards/margins": 0.13548563420772552, + "rewards/rejected": -0.2111039161682129, + "step": 5710 + }, + { + "epoch": 0.37, + "learning_rate": 3.9395095371115935e-06, + "logits/chosen": -2.3947396278381348, + "logits/rejected": -2.108790874481201, + "logps/chosen": -220.9462890625, + "logps/rejected": -224.45059204101562, + "loss": 1974.1141, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.08289176225662231, + "rewards/margins": 0.1411549299955368, + "rewards/rejected": -0.22404666244983673, + "step": 5720 + }, + { + "epoch": 0.37, + "learning_rate": 3.93483743302171e-06, + "logits/chosen": -2.3456485271453857, + "logits/rejected": -2.112895965576172, + "logps/chosen": -217.47824096679688, + "logps/rejected": -206.06838989257812, + "loss": 1963.5992, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.04686587676405907, + "rewards/margins": 0.12325469404459, + "rewards/rejected": -0.17012058198451996, + "step": 5730 + }, + { + "epoch": 0.38, + "learning_rate": 3.930157844116913e-06, + "logits/chosen": -2.19012451171875, + "logits/rejected": -2.1317851543426514, + "logps/chosen": -215.0693817138672, + "logps/rejected": -210.9638671875, + "loss": 2180.9488, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.053120873868465424, + "rewards/margins": 0.1061321273446083, + "rewards/rejected": -0.15925300121307373, + "step": 5740 + }, + { + "epoch": 0.38, + "learning_rate": 3.925470794808229e-06, + "logits/chosen": -2.343135356903076, + "logits/rejected": -1.9810895919799805, + "logps/chosen": -245.66616821289062, + "logps/rejected": -228.5177764892578, + "loss": 1596.3757, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06678466498851776, + "rewards/margins": 0.15668952465057373, + "rewards/rejected": -0.2234741747379303, + "step": 5750 + }, + { + "epoch": 0.38, + "learning_rate": 3.920776309545606e-06, + "logits/chosen": -2.4004764556884766, + "logits/rejected": -2.2372593879699707, + "logps/chosen": -153.7781982421875, + "logps/rejected": -159.7616729736328, + "loss": 1799.1465, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.010711869224905968, + "rewards/margins": 0.12815974652767181, + "rewards/rejected": -0.13887162506580353, + "step": 5760 + }, + { + "epoch": 0.38, + "learning_rate": 3.916074412817778e-06, + "logits/chosen": -2.2932066917419434, + "logits/rejected": -1.9272019863128662, + "logps/chosen": -250.5941925048828, + "logps/rejected": -271.01593017578125, + "loss": 1309.2438, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07260191440582275, + "rewards/margins": 0.19899395108222961, + "rewards/rejected": -0.27159583568573, + "step": 5770 + }, + { + "epoch": 0.38, + "learning_rate": 3.911365129152139e-06, + "logits/chosen": -2.42976713180542, + "logits/rejected": -2.275913715362549, + "logps/chosen": -238.23876953125, + "logps/rejected": -248.1154327392578, + "loss": 1842.8461, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.05804906040430069, + "rewards/margins": 0.1541648805141449, + "rewards/rejected": -0.21221396327018738, + "step": 5780 + }, + { + "epoch": 0.38, + "learning_rate": 3.906648483114623e-06, + "logits/chosen": -2.3079278469085693, + "logits/rejected": -2.209843158721924, + "logps/chosen": -208.1516876220703, + "logps/rejected": -193.66856384277344, + "loss": 1672.7014, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10828059911727905, + "rewards/margins": 0.1810188740491867, + "rewards/rejected": -0.28929945826530457, + "step": 5790 + }, + { + "epoch": 0.38, + "learning_rate": 3.901924499309564e-06, + "logits/chosen": -2.258023738861084, + "logits/rejected": -2.0684800148010254, + "logps/chosen": -237.38821411132812, + "logps/rejected": -218.6651611328125, + "loss": 1948.3748, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08683280646800995, + "rewards/margins": 0.13882941007614136, + "rewards/rejected": -0.2256622016429901, + "step": 5800 + }, + { + "epoch": 0.38, + "eval_logits/chosen": -2.346818208694458, + "eval_logits/rejected": -2.161313056945801, + "eval_logps/chosen": -240.79034423828125, + "eval_logps/rejected": -235.4921875, + "eval_loss": 1821.662353515625, + "eval_rewards/accuracies": 0.6549999713897705, + "eval_rewards/chosen": -0.08785369992256165, + "eval_rewards/margins": 0.15094910562038422, + "eval_rewards/rejected": -0.23880282044410706, + "eval_runtime": 724.1961, + "eval_samples_per_second": 2.762, + "eval_steps_per_second": 1.381, + "step": 5800 + }, + { + "epoch": 0.38, + "learning_rate": 3.897193202379575e-06, + "logits/chosen": -2.3440444469451904, + "logits/rejected": -2.175175905227661, + "logps/chosen": -207.85714721679688, + "logps/rejected": -207.29922485351562, + "loss": 1816.1107, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07379438728094101, + "rewards/margins": 0.16414874792099, + "rewards/rejected": -0.2379431426525116, + "step": 5810 + }, + { + "epoch": 0.38, + "learning_rate": 3.8924546170054215e-06, + "logits/chosen": -2.2868430614471436, + "logits/rejected": -2.2411949634552, + "logps/chosen": -223.8891143798828, + "logps/rejected": -219.9317169189453, + "loss": 1833.4615, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.07981501519680023, + "rewards/margins": 0.1551598608493805, + "rewards/rejected": -0.23497489094734192, + "step": 5820 + }, + { + "epoch": 0.38, + "learning_rate": 3.887708767905883e-06, + "logits/chosen": -2.5371270179748535, + "logits/rejected": -2.1372833251953125, + "logps/chosen": -253.84024047851562, + "logps/rejected": -200.68722534179688, + "loss": 1781.9459, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10073025524616241, + "rewards/margins": 0.12689939141273499, + "rewards/rejected": -0.2276296317577362, + "step": 5830 + }, + { + "epoch": 0.38, + "learning_rate": 3.882955679837636e-06, + "logits/chosen": -2.2934184074401855, + "logits/rejected": -2.178283929824829, + "logps/chosen": -254.0140380859375, + "logps/rejected": -270.5880126953125, + "loss": 1916.2676, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14637038111686707, + "rewards/margins": 0.1419863998889923, + "rewards/rejected": -0.2883567810058594, + "step": 5840 + }, + { + "epoch": 0.38, + "learning_rate": 3.878195377595113e-06, + "logits/chosen": -2.3741116523742676, + "logits/rejected": -2.1962015628814697, + "logps/chosen": -241.71084594726562, + "logps/rejected": -255.45974731445312, + "loss": 1891.0613, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.05805094167590141, + "rewards/margins": 0.19312414526939392, + "rewards/rejected": -0.2511751055717468, + "step": 5850 + }, + { + "epoch": 0.38, + "learning_rate": 3.873427886010384e-06, + "logits/chosen": -2.318110942840576, + "logits/rejected": -2.1745448112487793, + "logps/chosen": -197.3480224609375, + "logps/rejected": -193.17356872558594, + "loss": 1736.1059, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.06896080821752548, + "rewards/margins": 0.14914639294147491, + "rewards/rejected": -0.2181071788072586, + "step": 5860 + }, + { + "epoch": 0.38, + "learning_rate": 3.868653229953021e-06, + "logits/chosen": -2.3854384422302246, + "logits/rejected": -2.162374258041382, + "logps/chosen": -244.8651885986328, + "logps/rejected": -265.3497619628906, + "loss": 1352.0979, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.07650326192378998, + "rewards/margins": 0.22548480331897736, + "rewards/rejected": -0.30198806524276733, + "step": 5870 + }, + { + "epoch": 0.38, + "learning_rate": 3.8638714343299675e-06, + "logits/chosen": -2.2928366661071777, + "logits/rejected": -2.210357189178467, + "logps/chosen": -227.2082977294922, + "logps/rejected": -253.7676239013672, + "loss": 1918.273, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.08865579217672348, + "rewards/margins": 0.11443565785884857, + "rewards/rejected": -0.20309145748615265, + "step": 5880 + }, + { + "epoch": 0.39, + "learning_rate": 3.859082524085414e-06, + "logits/chosen": -2.3190252780914307, + "logits/rejected": -1.9635696411132812, + "logps/chosen": -279.509521484375, + "logps/rejected": -232.806884765625, + "loss": 1930.0092, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07166347652673721, + "rewards/margins": 0.13536502420902252, + "rewards/rejected": -0.20702850818634033, + "step": 5890 + }, + { + "epoch": 0.39, + "learning_rate": 3.854286524200659e-06, + "logits/chosen": -2.4598076343536377, + "logits/rejected": -2.2010176181793213, + "logps/chosen": -285.62982177734375, + "logps/rejected": -242.8809051513672, + "loss": 2247.3281, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.09212378412485123, + "rewards/margins": 0.07887519896030426, + "rewards/rejected": -0.1709989756345749, + "step": 5900 + }, + { + "epoch": 0.39, + "eval_logits/chosen": -2.3276898860931396, + "eval_logits/rejected": -2.144029140472412, + "eval_logps/chosen": -241.37901306152344, + "eval_logps/rejected": -234.9075469970703, + "eval_loss": 1835.3883056640625, + "eval_rewards/accuracies": 0.6539999842643738, + "eval_rewards/chosen": -0.09374046325683594, + "eval_rewards/margins": 0.1392161101102829, + "eval_rewards/rejected": -0.23295657336711884, + "eval_runtime": 724.7915, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 5900 + }, + { + "epoch": 0.39, + "learning_rate": 3.849483459693991e-06, + "logits/chosen": -2.4043338298797607, + "logits/rejected": -2.275143623352051, + "logps/chosen": -219.0234375, + "logps/rejected": -196.8841094970703, + "loss": 1608.4016, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09346772730350494, + "rewards/margins": 0.1652974635362625, + "rewards/rejected": -0.25876516103744507, + "step": 5910 + }, + { + "epoch": 0.39, + "learning_rate": 3.844673355620544e-06, + "logits/chosen": -2.291963577270508, + "logits/rejected": -2.1587347984313965, + "logps/chosen": -258.6552734375, + "logps/rejected": -235.81570434570312, + "loss": 1736.4279, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12082358449697495, + "rewards/margins": 0.1544593870639801, + "rewards/rejected": -0.27528297901153564, + "step": 5920 + }, + { + "epoch": 0.39, + "learning_rate": 3.839856237072178e-06, + "logits/chosen": -2.134535551071167, + "logits/rejected": -2.1370232105255127, + "logps/chosen": -195.16397094726562, + "logps/rejected": -224.6689453125, + "loss": 1963.8773, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1258985698223114, + "rewards/margins": 0.12312869727611542, + "rewards/rejected": -0.249027281999588, + "step": 5930 + }, + { + "epoch": 0.39, + "learning_rate": 3.8350321291773455e-06, + "logits/chosen": -2.064373016357422, + "logits/rejected": -2.0094499588012695, + "logps/chosen": -205.4337921142578, + "logps/rejected": -178.10940551757812, + "loss": 1742.5545, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.02911827340722084, + "rewards/margins": 0.16871920228004456, + "rewards/rejected": -0.1978374719619751, + "step": 5940 + }, + { + "epoch": 0.39, + "learning_rate": 3.830201057100953e-06, + "logits/chosen": -2.3538615703582764, + "logits/rejected": -2.3942766189575195, + "logps/chosen": -203.4180908203125, + "logps/rejected": -234.2006378173828, + "loss": 1670.6439, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11645101010799408, + "rewards/margins": 0.15289771556854248, + "rewards/rejected": -0.26934871077537537, + "step": 5950 + }, + { + "epoch": 0.39, + "learning_rate": 3.82536304604424e-06, + "logits/chosen": -2.1328577995300293, + "logits/rejected": -2.0833983421325684, + "logps/chosen": -245.0559539794922, + "logps/rejected": -227.7585906982422, + "loss": 1808.9207, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08891718089580536, + "rewards/margins": 0.14438779652118683, + "rewards/rejected": -0.2333049774169922, + "step": 5960 + }, + { + "epoch": 0.39, + "learning_rate": 3.8205181212446435e-06, + "logits/chosen": -2.498424530029297, + "logits/rejected": -2.272063732147217, + "logps/chosen": -275.7486267089844, + "logps/rejected": -250.5533447265625, + "loss": 1748.2926, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.051897335797548294, + "rewards/margins": 0.1520821750164032, + "rewards/rejected": -0.2039795219898224, + "step": 5970 + }, + { + "epoch": 0.39, + "learning_rate": 3.815666307975664e-06, + "logits/chosen": -2.3104419708251953, + "logits/rejected": -2.1840157508850098, + "logps/chosen": -249.99752807617188, + "logps/rejected": -244.8012237548828, + "loss": 1731.6719, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1267974078655243, + "rewards/margins": 0.13985633850097656, + "rewards/rejected": -0.26665374636650085, + "step": 5980 + }, + { + "epoch": 0.39, + "learning_rate": 3.8108076315467346e-06, + "logits/chosen": -2.3855650424957275, + "logits/rejected": -2.255192279815674, + "logps/chosen": -277.58660888671875, + "logps/rejected": -216.82421875, + "loss": 1885.3119, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.20303857326507568, + "rewards/margins": 0.13964101672172546, + "rewards/rejected": -0.34267958998680115, + "step": 5990 + }, + { + "epoch": 0.39, + "learning_rate": 3.805942117303093e-06, + "logits/chosen": -2.5637707710266113, + "logits/rejected": -2.2310338020324707, + "logps/chosen": -333.25775146484375, + "logps/rejected": -303.8687744140625, + "loss": 1582.7374, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.17222166061401367, + "rewards/margins": 0.17134621739387512, + "rewards/rejected": -0.3435679078102112, + "step": 6000 + }, + { + "epoch": 0.39, + "eval_logits/chosen": -2.3215813636779785, + "eval_logits/rejected": -2.1373915672302246, + "eval_logps/chosen": -251.67835998535156, + "eval_logps/rejected": -245.36856079101562, + "eval_loss": 1837.457763671875, + "eval_rewards/accuracies": 0.6545000076293945, + "eval_rewards/chosen": -0.19673386216163635, + "eval_rewards/margins": 0.1408328264951706, + "eval_rewards/rejected": -0.33756670355796814, + "eval_runtime": 724.3827, + "eval_samples_per_second": 2.761, + "eval_steps_per_second": 1.38, + "step": 6000 + }, + { + "epoch": 0.39, + "learning_rate": 3.8010697906256446e-06, + "logits/chosen": -2.1542887687683105, + "logits/rejected": -2.1475603580474854, + "logps/chosen": -233.7429656982422, + "logps/rejected": -216.17855834960938, + "loss": 2242.7635, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.27046507596969604, + "rewards/margins": 0.1026582345366478, + "rewards/rejected": -0.37312331795692444, + "step": 6010 + }, + { + "epoch": 0.39, + "learning_rate": 3.7961906769308323e-06, + "logits/chosen": -2.209507465362549, + "logits/rejected": -2.02146315574646, + "logps/chosen": -232.28689575195312, + "logps/rejected": -254.2529754638672, + "loss": 1794.5312, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.26051703095436096, + "rewards/margins": 0.11891166865825653, + "rewards/rejected": -0.3794287145137787, + "step": 6020 + }, + { + "epoch": 0.39, + "learning_rate": 3.7913048016705028e-06, + "logits/chosen": -2.253190040588379, + "logits/rejected": -2.2125744819641113, + "logps/chosen": -283.858642578125, + "logps/rejected": -293.50457763671875, + "loss": 2045.3717, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.239190936088562, + "rewards/margins": 0.11693974584341049, + "rewards/rejected": -0.3561306893825531, + "step": 6030 + }, + { + "epoch": 0.4, + "learning_rate": 3.786412190331775e-06, + "logits/chosen": -2.465777635574341, + "logits/rejected": -2.146012544631958, + "logps/chosen": -219.91726684570312, + "logps/rejected": -193.1280975341797, + "loss": 2000.1277, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.17056769132614136, + "rewards/margins": 0.10310175269842148, + "rewards/rejected": -0.27366942167282104, + "step": 6040 + }, + { + "epoch": 0.4, + "learning_rate": 3.781512868436906e-06, + "logits/chosen": -2.464073896408081, + "logits/rejected": -2.3181827068328857, + "logps/chosen": -147.85577392578125, + "logps/rejected": -168.68588256835938, + "loss": 1561.3501, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1524503231048584, + "rewards/margins": 0.13041134178638458, + "rewards/rejected": -0.2828616499900818, + "step": 6050 + }, + { + "epoch": 0.4, + "learning_rate": 3.7766068615431605e-06, + "logits/chosen": -2.287479877471924, + "logits/rejected": -2.2375855445861816, + "logps/chosen": -275.55511474609375, + "logps/rejected": -237.14199829101562, + "loss": 2315.9816, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.18468059599399567, + "rewards/margins": 0.10221859067678452, + "rewards/rejected": -0.2868991792201996, + "step": 6060 + }, + { + "epoch": 0.4, + "learning_rate": 3.771694195242671e-06, + "logits/chosen": -2.5044009685516357, + "logits/rejected": -2.073282480239868, + "logps/chosen": -314.542236328125, + "logps/rejected": -218.26248168945312, + "loss": 1707.1232, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15363477170467377, + "rewards/margins": 0.15451176464557648, + "rewards/rejected": -0.30814653635025024, + "step": 6070 + }, + { + "epoch": 0.4, + "learning_rate": 3.766774895162314e-06, + "logits/chosen": -2.3283276557922363, + "logits/rejected": -2.2265541553497314, + "logps/chosen": -265.23028564453125, + "logps/rejected": -218.9558868408203, + "loss": 2090.2014, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1869053840637207, + "rewards/margins": 0.09861340373754501, + "rewards/rejected": -0.2855187654495239, + "step": 6080 + }, + { + "epoch": 0.4, + "learning_rate": 3.7618489869635666e-06, + "logits/chosen": -2.2331745624542236, + "logits/rejected": -2.183838129043579, + "logps/chosen": -272.5234069824219, + "logps/rejected": -266.155517578125, + "loss": 1883.102, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2549645006656647, + "rewards/margins": 0.13375045359134674, + "rewards/rejected": -0.388714998960495, + "step": 6090 + }, + { + "epoch": 0.4, + "learning_rate": 3.756916496342379e-06, + "logits/chosen": -2.2571969032287598, + "logits/rejected": -2.3134007453918457, + "logps/chosen": -212.2631072998047, + "logps/rejected": -232.08206176757812, + "loss": 2022.2906, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.2539118826389313, + "rewards/margins": 0.11366178840398788, + "rewards/rejected": -0.36757364869117737, + "step": 6100 + }, + { + "epoch": 0.4, + "eval_logits/chosen": -2.3228988647460938, + "eval_logits/rejected": -2.138225793838501, + "eval_logps/chosen": -255.5701446533203, + "eval_logps/rejected": -247.9512939453125, + "eval_loss": 1846.5860595703125, + "eval_rewards/accuracies": 0.6510000228881836, + "eval_rewards/chosen": -0.23565174639225006, + "eval_rewards/margins": 0.1277419775724411, + "eval_rewards/rejected": -0.3633936941623688, + "eval_runtime": 722.4831, + "eval_samples_per_second": 2.768, + "eval_steps_per_second": 1.384, + "step": 6100 + }, + { + "epoch": 0.4, + "learning_rate": 3.751977449029039e-06, + "logits/chosen": -1.982560396194458, + "logits/rejected": -1.9873220920562744, + "logps/chosen": -285.7325439453125, + "logps/rejected": -259.8460693359375, + "loss": 2041.9143, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2565609812736511, + "rewards/margins": 0.1150914877653122, + "rewards/rejected": -0.3716525435447693, + "step": 6110 + }, + { + "epoch": 0.4, + "learning_rate": 3.747031870788037e-06, + "logits/chosen": -2.4981637001037598, + "logits/rejected": -2.1473708152770996, + "logps/chosen": -337.43927001953125, + "logps/rejected": -270.34967041015625, + "loss": 1604.6782, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.2068602293729782, + "rewards/margins": 0.1532965451478958, + "rewards/rejected": -0.3601568043231964, + "step": 6120 + }, + { + "epoch": 0.4, + "learning_rate": 3.7420797874179326e-06, + "logits/chosen": -2.273043394088745, + "logits/rejected": -2.018402338027954, + "logps/chosen": -259.5672302246094, + "logps/rejected": -209.3104705810547, + "loss": 2124.3014, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.22457745671272278, + "rewards/margins": 0.08632310479879379, + "rewards/rejected": -0.31090056896209717, + "step": 6130 + }, + { + "epoch": 0.4, + "learning_rate": 3.7371212247512167e-06, + "logits/chosen": -2.600759983062744, + "logits/rejected": -2.2669732570648193, + "logps/chosen": -338.55413818359375, + "logps/rejected": -288.6649475097656, + "loss": 1795.9818, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12159669399261475, + "rewards/margins": 0.15219008922576904, + "rewards/rejected": -0.2737867832183838, + "step": 6140 + }, + { + "epoch": 0.4, + "learning_rate": 3.7321562086541817e-06, + "logits/chosen": -2.376091480255127, + "logits/rejected": -2.273066997528076, + "logps/chosen": -271.08477783203125, + "logps/rejected": -285.23175048828125, + "loss": 1733.2641, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1810462772846222, + "rewards/margins": 0.13356760144233704, + "rewards/rejected": -0.3146139085292816, + "step": 6150 + }, + { + "epoch": 0.4, + "learning_rate": 3.7271847650267834e-06, + "logits/chosen": -2.1715023517608643, + "logits/rejected": -2.0725510120391846, + "logps/chosen": -220.02145385742188, + "logps/rejected": -229.8434295654297, + "loss": 1948.7779, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2008678913116455, + "rewards/margins": 0.10712286084890366, + "rewards/rejected": -0.30799075961112976, + "step": 6160 + }, + { + "epoch": 0.4, + "learning_rate": 3.7222069198025086e-06, + "logits/chosen": -2.1698288917541504, + "logits/rejected": -1.9773457050323486, + "logps/chosen": -231.421142578125, + "logps/rejected": -222.72280883789062, + "loss": 1984.9541, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.228786900639534, + "rewards/margins": 0.11893894523382187, + "rewards/rejected": -0.34772583842277527, + "step": 6170 + }, + { + "epoch": 0.4, + "learning_rate": 3.7172226989482353e-06, + "logits/chosen": -2.1827809810638428, + "logits/rejected": -1.970580816268921, + "logps/chosen": -228.5430145263672, + "logps/rejected": -239.3963165283203, + "loss": 1866.3375, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.18570196628570557, + "rewards/margins": 0.1299847662448883, + "rewards/rejected": -0.31568676233291626, + "step": 6180 + }, + { + "epoch": 0.4, + "learning_rate": 3.7122321284641007e-06, + "logits/chosen": -2.4312362670898438, + "logits/rejected": -2.0049402713775635, + "logps/chosen": -386.119140625, + "logps/rejected": -289.6749572753906, + "loss": 1797.6766, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2333250492811203, + "rewards/margins": 0.14045672118663788, + "rewards/rejected": -0.3737817704677582, + "step": 6190 + }, + { + "epoch": 0.41, + "learning_rate": 3.707235234383365e-06, + "logits/chosen": -2.3579580783843994, + "logits/rejected": -2.014785051345825, + "logps/chosen": -274.70660400390625, + "logps/rejected": -202.1422882080078, + "loss": 1575.1918, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1345268189907074, + "rewards/margins": 0.14214524626731873, + "rewards/rejected": -0.27667203545570374, + "step": 6200 + }, + { + "epoch": 0.41, + "eval_logits/chosen": -2.317455530166626, + "eval_logits/rejected": -2.133312940597534, + "eval_logps/chosen": -246.5572052001953, + "eval_logps/rejected": -241.9267120361328, + "eval_loss": 1803.5977783203125, + "eval_rewards/accuracies": 0.6604999899864197, + "eval_rewards/chosen": -0.14552246034145355, + "eval_rewards/margins": 0.1576259285211563, + "eval_rewards/rejected": -0.30314841866493225, + "eval_runtime": 726.1251, + "eval_samples_per_second": 2.754, + "eval_steps_per_second": 1.377, + "step": 6200 + }, + { + "epoch": 0.41, + "learning_rate": 3.702232042772277e-06, + "logits/chosen": -2.160836696624756, + "logits/rejected": -2.10648250579834, + "logps/chosen": -221.47750854492188, + "logps/rejected": -217.98605346679688, + "loss": 1523.7708, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18931357562541962, + "rewards/margins": 0.1886318176984787, + "rewards/rejected": -0.3779454231262207, + "step": 6210 + }, + { + "epoch": 0.41, + "learning_rate": 3.6972225797299325e-06, + "logits/chosen": -2.270308017730713, + "logits/rejected": -2.3070664405822754, + "logps/chosen": -266.1285095214844, + "logps/rejected": -267.2669982910156, + "loss": 1845.6088, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.15619874000549316, + "rewards/margins": 0.17074045538902283, + "rewards/rejected": -0.326939195394516, + "step": 6220 + }, + { + "epoch": 0.41, + "learning_rate": 3.692206871388147e-06, + "logits/chosen": -2.3865723609924316, + "logits/rejected": -1.942652940750122, + "logps/chosen": -247.4679718017578, + "logps/rejected": -225.7694091796875, + "loss": 1537.6834, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15079227089881897, + "rewards/margins": 0.19530631601810455, + "rewards/rejected": -0.34609857201576233, + "step": 6230 + }, + { + "epoch": 0.41, + "learning_rate": 3.6871849439113115e-06, + "logits/chosen": -2.097397565841675, + "logits/rejected": -2.0197715759277344, + "logps/chosen": -247.7275848388672, + "logps/rejected": -250.33688354492188, + "loss": 2091.8342, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17816750705242157, + "rewards/margins": 0.12157128006219864, + "rewards/rejected": -0.2997387945652008, + "step": 6240 + }, + { + "epoch": 0.41, + "learning_rate": 3.682156823496259e-06, + "logits/chosen": -2.333956241607666, + "logits/rejected": -2.0595738887786865, + "logps/chosen": -233.3226318359375, + "logps/rejected": -220.8282012939453, + "loss": 1475.5063, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1940866857767105, + "rewards/margins": 0.19871152937412262, + "rewards/rejected": -0.39279818534851074, + "step": 6250 + }, + { + "epoch": 0.41, + "learning_rate": 3.67712253637213e-06, + "logits/chosen": -2.3630833625793457, + "logits/rejected": -2.1345248222351074, + "logps/chosen": -313.71661376953125, + "logps/rejected": -241.8723602294922, + "loss": 1499.583, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2142215520143509, + "rewards/margins": 0.180682972073555, + "rewards/rejected": -0.3949044644832611, + "step": 6260 + }, + { + "epoch": 0.41, + "learning_rate": 3.672082108800231e-06, + "logits/chosen": -2.1950974464416504, + "logits/rejected": -1.9880892038345337, + "logps/chosen": -248.4373016357422, + "logps/rejected": -224.557373046875, + "loss": 2001.3568, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3462149202823639, + "rewards/margins": 0.10159517824649811, + "rewards/rejected": -0.44781002402305603, + "step": 6270 + }, + { + "epoch": 0.41, + "learning_rate": 3.6670355670739012e-06, + "logits/chosen": -2.308696746826172, + "logits/rejected": -2.1200101375579834, + "logps/chosen": -181.59396362304688, + "logps/rejected": -195.4339141845703, + "loss": 1520.7145, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19640548527240753, + "rewards/margins": 0.16478656232357025, + "rewards/rejected": -0.3611920475959778, + "step": 6280 + }, + { + "epoch": 0.41, + "learning_rate": 3.6619829375183745e-06, + "logits/chosen": -2.470808506011963, + "logits/rejected": -2.2788443565368652, + "logps/chosen": -244.7198028564453, + "logps/rejected": -246.4285125732422, + "loss": 1809.3705, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18717266619205475, + "rewards/margins": 0.17779619991779327, + "rewards/rejected": -0.36496883630752563, + "step": 6290 + }, + { + "epoch": 0.41, + "learning_rate": 3.6569242464906427e-06, + "logits/chosen": -2.3372724056243896, + "logits/rejected": -2.159214496612549, + "logps/chosen": -217.8628692626953, + "logps/rejected": -258.60821533203125, + "loss": 1673.5182, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.10133813321590424, + "rewards/margins": 0.1969621181488037, + "rewards/rejected": -0.29830026626586914, + "step": 6300 + }, + { + "epoch": 0.41, + "eval_logits/chosen": -2.3376824855804443, + "eval_logits/rejected": -2.151590347290039, + "eval_logps/chosen": -244.78695678710938, + "eval_logps/rejected": -240.61180114746094, + "eval_loss": 1812.673828125, + "eval_rewards/accuracies": 0.656499981880188, + "eval_rewards/chosen": -0.1278199702501297, + "eval_rewards/margins": 0.16217908263206482, + "eval_rewards/rejected": -0.2899990975856781, + "eval_runtime": 722.487, + "eval_samples_per_second": 2.768, + "eval_steps_per_second": 1.384, + "step": 6300 + }, + { + "epoch": 0.41, + "learning_rate": 3.6518595203793156e-06, + "logits/chosen": -2.178379774093628, + "logits/rejected": -2.198406934738159, + "logps/chosen": -273.59149169921875, + "logps/rejected": -306.0458984375, + "loss": 1618.5656, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.148772194981575, + "rewards/margins": 0.18979643285274506, + "rewards/rejected": -0.33856862783432007, + "step": 6310 + }, + { + "epoch": 0.41, + "learning_rate": 3.646788785604485e-06, + "logits/chosen": -2.2992780208587646, + "logits/rejected": -2.1557576656341553, + "logps/chosen": -221.5495147705078, + "logps/rejected": -232.4991455078125, + "loss": 2059.0676, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1589115560054779, + "rewards/margins": 0.1319461166858673, + "rewards/rejected": -0.2908576726913452, + "step": 6320 + }, + { + "epoch": 0.41, + "learning_rate": 3.641712068617588e-06, + "logits/chosen": -2.323676347732544, + "logits/rejected": -2.2105374336242676, + "logps/chosen": -276.9544677734375, + "logps/rejected": -239.6781005859375, + "loss": 1482.207, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1559438407421112, + "rewards/margins": 0.16268186271190643, + "rewards/rejected": -0.31862568855285645, + "step": 6330 + }, + { + "epoch": 0.41, + "learning_rate": 3.6366293959012673e-06, + "logits/chosen": -2.2658047676086426, + "logits/rejected": -2.0348846912384033, + "logps/chosen": -194.25845336914062, + "logps/rejected": -192.47836303710938, + "loss": 1675.3195, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1305646449327469, + "rewards/margins": 0.17743127048015594, + "rewards/rejected": -0.30799591541290283, + "step": 6340 + }, + { + "epoch": 0.42, + "learning_rate": 3.631540793969233e-06, + "logits/chosen": -2.401700258255005, + "logits/rejected": -2.3811542987823486, + "logps/chosen": -197.39588928222656, + "logps/rejected": -212.9019012451172, + "loss": 1736.9973, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08778167515993118, + "rewards/margins": 0.13827702403068542, + "rewards/rejected": -0.22605867683887482, + "step": 6350 + }, + { + "epoch": 0.42, + "learning_rate": 3.626446289366127e-06, + "logits/chosen": -2.5857996940612793, + "logits/rejected": -2.146826982498169, + "logps/chosen": -235.07467651367188, + "logps/rejected": -176.82415771484375, + "loss": 1670.159, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1492549479007721, + "rewards/margins": 0.1326829344034195, + "rewards/rejected": -0.281937837600708, + "step": 6360 + }, + { + "epoch": 0.42, + "learning_rate": 3.6213459086673786e-06, + "logits/chosen": -2.3118743896484375, + "logits/rejected": -2.372246742248535, + "logps/chosen": -182.10699462890625, + "logps/rejected": -205.96987915039062, + "loss": 2017.7496, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11169429123401642, + "rewards/margins": 0.14489558339118958, + "rewards/rejected": -0.2565898597240448, + "step": 6370 + }, + { + "epoch": 0.42, + "learning_rate": 3.6162396784790737e-06, + "logits/chosen": -2.2020115852355957, + "logits/rejected": -2.2304563522338867, + "logps/chosen": -232.52236938476562, + "logps/rejected": -243.04067993164062, + "loss": 2114.6471, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.09515275061130524, + "rewards/margins": 0.10677912086248398, + "rewards/rejected": -0.20193186402320862, + "step": 6380 + }, + { + "epoch": 0.42, + "learning_rate": 3.6111276254378095e-06, + "logits/chosen": -2.28242564201355, + "logits/rejected": -2.2833101749420166, + "logps/chosen": -234.3068084716797, + "logps/rejected": -247.7659912109375, + "loss": 1578.4936, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06349892169237137, + "rewards/margins": 0.17872732877731323, + "rewards/rejected": -0.242226243019104, + "step": 6390 + }, + { + "epoch": 0.42, + "learning_rate": 3.606009776210559e-06, + "logits/chosen": -2.269806146621704, + "logits/rejected": -2.038888454437256, + "logps/chosen": -256.7181091308594, + "logps/rejected": -229.2235870361328, + "loss": 1820.5289, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1609443873167038, + "rewards/margins": 0.16164250671863556, + "rewards/rejected": -0.32258686423301697, + "step": 6400 + }, + { + "epoch": 0.42, + "eval_logits/chosen": -2.301616668701172, + "eval_logits/rejected": -2.1188178062438965, + "eval_logps/chosen": -245.5502471923828, + "eval_logps/rejected": -240.75802612304688, + "eval_loss": 1817.4549560546875, + "eval_rewards/accuracies": 0.6514999866485596, + "eval_rewards/chosen": -0.13545304536819458, + "eval_rewards/margins": 0.15600834786891937, + "eval_rewards/rejected": -0.29146140813827515, + "eval_runtime": 723.5291, + "eval_samples_per_second": 2.764, + "eval_steps_per_second": 1.382, + "step": 6400 + }, + { + "epoch": 0.42, + "learning_rate": 3.600886157494531e-06, + "logits/chosen": -2.3986454010009766, + "logits/rejected": -2.279700517654419, + "logps/chosen": -276.1898193359375, + "logps/rejected": -282.24639892578125, + "loss": 1617.5596, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13333269953727722, + "rewards/margins": 0.1783868670463562, + "rewards/rejected": -0.3117195963859558, + "step": 6410 + }, + { + "epoch": 0.42, + "learning_rate": 3.5957567960170304e-06, + "logits/chosen": -2.485504627227783, + "logits/rejected": -1.7878259420394897, + "logps/chosen": -303.04351806640625, + "logps/rejected": -204.10830688476562, + "loss": 1415.637, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11858425289392471, + "rewards/margins": 0.1972246915102005, + "rewards/rejected": -0.3158089518547058, + "step": 6420 + }, + { + "epoch": 0.42, + "learning_rate": 3.590621718535319e-06, + "logits/chosen": -2.114732503890991, + "logits/rejected": -1.928137183189392, + "logps/chosen": -216.5114288330078, + "logps/rejected": -232.0654754638672, + "loss": 1787.9346, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1978907436132431, + "rewards/margins": 0.16446657478809357, + "rewards/rejected": -0.3623572885990143, + "step": 6430 + }, + { + "epoch": 0.42, + "learning_rate": 3.5854809518364775e-06, + "logits/chosen": -2.3756461143493652, + "logits/rejected": -2.071209192276001, + "logps/chosen": -254.34521484375, + "logps/rejected": -230.24819946289062, + "loss": 1597.4032, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.08198460191488266, + "rewards/margins": 0.20415684580802917, + "rewards/rejected": -0.2861414849758148, + "step": 6440 + }, + { + "epoch": 0.42, + "learning_rate": 3.580334522737262e-06, + "logits/chosen": -2.2984519004821777, + "logits/rejected": -2.0156452655792236, + "logps/chosen": -217.0636749267578, + "logps/rejected": -197.22459411621094, + "loss": 1980.9646, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.15119247138500214, + "rewards/margins": 0.13810545206069946, + "rewards/rejected": -0.2892979383468628, + "step": 6450 + }, + { + "epoch": 0.42, + "learning_rate": 3.575182458083968e-06, + "logits/chosen": -2.239943265914917, + "logits/rejected": -2.189892530441284, + "logps/chosen": -254.2470245361328, + "logps/rejected": -242.56753540039062, + "loss": 1728.4674, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13441982865333557, + "rewards/margins": 0.16325482726097107, + "rewards/rejected": -0.29767465591430664, + "step": 6460 + }, + { + "epoch": 0.42, + "learning_rate": 3.5700247847522883e-06, + "logits/chosen": -2.3368194103240967, + "logits/rejected": -2.2533984184265137, + "logps/chosen": -213.6424102783203, + "logps/rejected": -231.5264434814453, + "loss": 1711.3686, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.10844433307647705, + "rewards/margins": 0.17902222275733948, + "rewards/rejected": -0.28746655583381653, + "step": 6470 + }, + { + "epoch": 0.42, + "learning_rate": 3.5648615296471743e-06, + "logits/chosen": -2.135636806488037, + "logits/rejected": -2.104698419570923, + "logps/chosen": -212.5499725341797, + "logps/rejected": -265.55413818359375, + "loss": 1671.6654, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15187156200408936, + "rewards/margins": 0.1493034064769745, + "rewards/rejected": -0.30117496848106384, + "step": 6480 + }, + { + "epoch": 0.42, + "learning_rate": 3.559692719702693e-06, + "logits/chosen": -2.1498494148254395, + "logits/rejected": -1.84506094455719, + "logps/chosen": -298.0892639160156, + "logps/rejected": -260.2900695800781, + "loss": 1792.5463, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11944104731082916, + "rewards/margins": 0.18930336833000183, + "rewards/rejected": -0.3087444007396698, + "step": 6490 + }, + { + "epoch": 0.43, + "learning_rate": 3.55451838188189e-06, + "logits/chosen": -2.268026828765869, + "logits/rejected": -2.1630935668945312, + "logps/chosen": -270.82183837890625, + "logps/rejected": -307.6367492675781, + "loss": 1858.2232, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.09535042196512222, + "rewards/margins": 0.148817777633667, + "rewards/rejected": -0.24416819214820862, + "step": 6500 + }, + { + "epoch": 0.43, + "eval_logits/chosen": -2.3055551052093506, + "eval_logits/rejected": -2.121851921081543, + "eval_logps/chosen": -246.60879516601562, + "eval_logps/rejected": -240.28997802734375, + "eval_loss": 1821.5572509765625, + "eval_rewards/accuracies": 0.6510000228881836, + "eval_rewards/chosen": -0.14603851735591888, + "eval_rewards/margins": 0.14074237644672394, + "eval_rewards/rejected": -0.2867808938026428, + "eval_runtime": 725.9926, + "eval_samples_per_second": 2.755, + "eval_steps_per_second": 1.377, + "step": 6500 + }, + { + "epoch": 0.43, + "learning_rate": 3.549338543176645e-06, + "logits/chosen": -2.314718723297119, + "logits/rejected": -2.0543487071990967, + "logps/chosen": -318.36981201171875, + "logps/rejected": -301.492431640625, + "loss": 1700.1051, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11881273984909058, + "rewards/margins": 0.15475775301456451, + "rewards/rejected": -0.2735705077648163, + "step": 6510 + }, + { + "epoch": 0.43, + "learning_rate": 3.5441532306075342e-06, + "logits/chosen": -2.2688286304473877, + "logits/rejected": -2.2558884620666504, + "logps/chosen": -247.38119506835938, + "logps/rejected": -306.28326416015625, + "loss": 2101.4018, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.15962006151676178, + "rewards/margins": 0.12336771190166473, + "rewards/rejected": -0.2829877734184265, + "step": 6520 + }, + { + "epoch": 0.43, + "learning_rate": 3.5389624712236894e-06, + "logits/chosen": -2.2822506427764893, + "logits/rejected": -2.0567774772644043, + "logps/chosen": -227.70010375976562, + "logps/rejected": -209.6324920654297, + "loss": 2123.4619, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.13664281368255615, + "rewards/margins": 0.08860232681035995, + "rewards/rejected": -0.22524511814117432, + "step": 6530 + }, + { + "epoch": 0.43, + "learning_rate": 3.533766292102653e-06, + "logits/chosen": -2.250833034515381, + "logits/rejected": -2.1126649379730225, + "logps/chosen": -226.72494506835938, + "logps/rejected": -228.54183959960938, + "loss": 1740.1117, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.16880393028259277, + "rewards/margins": 0.1350560188293457, + "rewards/rejected": -0.30385997891426086, + "step": 6540 + }, + { + "epoch": 0.43, + "learning_rate": 3.5285647203502404e-06, + "logits/chosen": -2.445377826690674, + "logits/rejected": -2.3247883319854736, + "logps/chosen": -269.4084167480469, + "logps/rejected": -244.4662628173828, + "loss": 1756.2137, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12137410789728165, + "rewards/margins": 0.12823879718780518, + "rewards/rejected": -0.24961289763450623, + "step": 6550 + }, + { + "epoch": 0.43, + "learning_rate": 3.5233577831003983e-06, + "logits/chosen": -2.277635097503662, + "logits/rejected": -2.130423069000244, + "logps/chosen": -265.00054931640625, + "logps/rejected": -251.6302032470703, + "loss": 1936.7143, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.12824609875679016, + "rewards/margins": 0.1029827818274498, + "rewards/rejected": -0.23122890293598175, + "step": 6560 + }, + { + "epoch": 0.43, + "learning_rate": 3.5181455075150628e-06, + "logits/chosen": -2.204834461212158, + "logits/rejected": -1.831272840499878, + "logps/chosen": -192.53721618652344, + "logps/rejected": -159.6791534423828, + "loss": 1779.9844, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10591045767068863, + "rewards/margins": 0.137648344039917, + "rewards/rejected": -0.24355880916118622, + "step": 6570 + }, + { + "epoch": 0.43, + "learning_rate": 3.512927920784016e-06, + "logits/chosen": -2.373384952545166, + "logits/rejected": -2.2336928844451904, + "logps/chosen": -234.6550750732422, + "logps/rejected": -231.44595336914062, + "loss": 1797.485, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.08980982005596161, + "rewards/margins": 0.1677091419696808, + "rewards/rejected": -0.2575189769268036, + "step": 6580 + }, + { + "epoch": 0.43, + "learning_rate": 3.5077050501247457e-06, + "logits/chosen": -2.461190700531006, + "logits/rejected": -2.015806198120117, + "logps/chosen": -287.93109130859375, + "logps/rejected": -236.8423309326172, + "loss": 1750.6951, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.06916189938783646, + "rewards/margins": 0.16299793124198914, + "rewards/rejected": -0.232159823179245, + "step": 6590 + }, + { + "epoch": 0.43, + "learning_rate": 3.5024769227823042e-06, + "logits/chosen": -2.3659586906433105, + "logits/rejected": -2.2356834411621094, + "logps/chosen": -178.3804473876953, + "logps/rejected": -149.21047973632812, + "loss": 2088.2963, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15509124100208282, + "rewards/margins": 0.11627724021673203, + "rewards/rejected": -0.27136847376823425, + "step": 6600 + }, + { + "epoch": 0.43, + "eval_logits/chosen": -2.349168300628662, + "eval_logits/rejected": -2.1621055603027344, + "eval_logps/chosen": -245.64903259277344, + "eval_logps/rejected": -239.01058959960938, + "eval_loss": 1821.9971923828125, + "eval_rewards/accuracies": 0.6474999785423279, + "eval_rewards/chosen": -0.13644078373908997, + "eval_rewards/margins": 0.13754619657993317, + "eval_rewards/rejected": -0.2739869952201843, + "eval_runtime": 725.2067, + "eval_samples_per_second": 2.758, + "eval_steps_per_second": 1.379, + "step": 6600 + }, + { + "epoch": 0.43, + "learning_rate": 3.4972435660291646e-06, + "logits/chosen": -2.4346280097961426, + "logits/rejected": -2.29125714302063, + "logps/chosen": -259.4081726074219, + "logps/rejected": -248.4931182861328, + "loss": 1667.1441, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16263893246650696, + "rewards/margins": 0.14024636149406433, + "rewards/rejected": -0.3028852939605713, + "step": 6610 + }, + { + "epoch": 0.43, + "learning_rate": 3.492005007165079e-06, + "logits/chosen": -2.3081727027893066, + "logits/rejected": -2.0565145015716553, + "logps/chosen": -242.61135864257812, + "logps/rejected": -259.98260498046875, + "loss": 2118.1311, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.13074350357055664, + "rewards/margins": 0.11214659363031387, + "rewards/rejected": -0.2428901195526123, + "step": 6620 + }, + { + "epoch": 0.43, + "learning_rate": 3.4867612735169377e-06, + "logits/chosen": -2.452394485473633, + "logits/rejected": -2.1109442710876465, + "logps/chosen": -240.4641876220703, + "logps/rejected": -180.43972778320312, + "loss": 1365.1011, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15429827570915222, + "rewards/margins": 0.182037353515625, + "rewards/rejected": -0.3363357186317444, + "step": 6630 + }, + { + "epoch": 0.43, + "learning_rate": 3.4815123924386226e-06, + "logits/chosen": -2.607165575027466, + "logits/rejected": -2.3029696941375732, + "logps/chosen": -325.56683349609375, + "logps/rejected": -266.5555725097656, + "loss": 1970.8203, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.19025450944900513, + "rewards/margins": 0.10661245882511139, + "rewards/rejected": -0.2968669533729553, + "step": 6640 + }, + { + "epoch": 0.44, + "learning_rate": 3.4762583913108696e-06, + "logits/chosen": -2.1845288276672363, + "logits/rejected": -1.9540420770645142, + "logps/chosen": -288.0469970703125, + "logps/rejected": -261.729736328125, + "loss": 1996.6184, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1828862875699997, + "rewards/margins": 0.1459795981645584, + "rewards/rejected": -0.3288659155368805, + "step": 6650 + }, + { + "epoch": 0.44, + "learning_rate": 3.4709992975411217e-06, + "logits/chosen": -2.304579019546509, + "logits/rejected": -1.8208246231079102, + "logps/chosen": -277.58221435546875, + "logps/rejected": -243.2197723388672, + "loss": 1692.5258, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2262284755706787, + "rewards/margins": 0.17904505133628845, + "rewards/rejected": -0.4052734971046448, + "step": 6660 + }, + { + "epoch": 0.44, + "learning_rate": 3.4657351385633886e-06, + "logits/chosen": -2.433800220489502, + "logits/rejected": -2.0775046348571777, + "logps/chosen": -204.56060791015625, + "logps/rejected": -209.0178985595703, + "loss": 1613.1366, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1926989108324051, + "rewards/margins": 0.1665751039981842, + "rewards/rejected": -0.3592740297317505, + "step": 6670 + }, + { + "epoch": 0.44, + "learning_rate": 3.4604659418381024e-06, + "logits/chosen": -2.34576153755188, + "logits/rejected": -1.9419485330581665, + "logps/chosen": -243.0738983154297, + "logps/rejected": -228.9387664794922, + "loss": 1889.6242, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.25019600987434387, + "rewards/margins": 0.1539548635482788, + "rewards/rejected": -0.4041508138179779, + "step": 6680 + }, + { + "epoch": 0.44, + "learning_rate": 3.4551917348519744e-06, + "logits/chosen": -2.4607691764831543, + "logits/rejected": -2.187936544418335, + "logps/chosen": -297.63330078125, + "logps/rejected": -264.26025390625, + "loss": 1638.1448, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1613462269306183, + "rewards/margins": 0.13827499747276306, + "rewards/rejected": -0.29962125420570374, + "step": 6690 + }, + { + "epoch": 0.44, + "learning_rate": 3.4499125451178505e-06, + "logits/chosen": -2.046312093734741, + "logits/rejected": -2.137775421142578, + "logps/chosen": -225.225341796875, + "logps/rejected": -260.2182312011719, + "loss": 1938.0561, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.20934048295021057, + "rewards/margins": 0.10380176454782486, + "rewards/rejected": -0.31314224004745483, + "step": 6700 + }, + { + "epoch": 0.44, + "eval_logits/chosen": -2.3692967891693115, + "eval_logits/rejected": -2.180675506591797, + "eval_logps/chosen": -251.37413024902344, + "eval_logps/rejected": -245.4427032470703, + "eval_loss": 1810.6575927734375, + "eval_rewards/accuracies": 0.6489999890327454, + "eval_rewards/chosen": -0.19369174540042877, + "eval_rewards/margins": 0.14461614191532135, + "eval_rewards/rejected": -0.33830785751342773, + "eval_runtime": 723.8586, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.381, + "step": 6700 + }, + { + "epoch": 0.44, + "learning_rate": 3.4446284001745723e-06, + "logits/chosen": -2.188316822052002, + "logits/rejected": -1.9344806671142578, + "logps/chosen": -230.5163116455078, + "logps/rejected": -260.1301574707031, + "loss": 1848.2039, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.242826908826828, + "rewards/margins": 0.1286681443452835, + "rewards/rejected": -0.3714950680732727, + "step": 6710 + }, + { + "epoch": 0.44, + "learning_rate": 3.439339327586827e-06, + "logits/chosen": -2.333066940307617, + "logits/rejected": -2.2780447006225586, + "logps/chosen": -203.406494140625, + "logps/rejected": -212.65017700195312, + "loss": 1913.4416, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.13388803601264954, + "rewards/margins": 0.14607299864292145, + "rewards/rejected": -0.2799610495567322, + "step": 6720 + }, + { + "epoch": 0.44, + "learning_rate": 3.434045354945008e-06, + "logits/chosen": -2.424826145172119, + "logits/rejected": -2.161849021911621, + "logps/chosen": -299.0447082519531, + "logps/rejected": -310.2620849609375, + "loss": 1639.3686, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2502012848854065, + "rewards/margins": 0.15953245759010315, + "rewards/rejected": -0.40973377227783203, + "step": 6730 + }, + { + "epoch": 0.44, + "learning_rate": 3.4287465098650713e-06, + "logits/chosen": -2.4182772636413574, + "logits/rejected": -2.4277045726776123, + "logps/chosen": -268.342529296875, + "logps/rejected": -267.02447509765625, + "loss": 1897.8031, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.20367029309272766, + "rewards/margins": 0.13523490726947784, + "rewards/rejected": -0.3389051556587219, + "step": 6740 + }, + { + "epoch": 0.44, + "learning_rate": 3.423442819988387e-06, + "logits/chosen": -2.2725331783294678, + "logits/rejected": -2.1603612899780273, + "logps/chosen": -208.27682495117188, + "logps/rejected": -214.9034881591797, + "loss": 2302.1482, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.24889573454856873, + "rewards/margins": 0.13816052675247192, + "rewards/rejected": -0.38705623149871826, + "step": 6750 + }, + { + "epoch": 0.44, + "learning_rate": 3.4181343129816e-06, + "logits/chosen": -2.4681544303894043, + "logits/rejected": -2.1562418937683105, + "logps/chosen": -190.3780059814453, + "logps/rejected": -188.13047790527344, + "loss": 2212.4684, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.14016306400299072, + "rewards/margins": 0.1204262375831604, + "rewards/rejected": -0.2605893015861511, + "step": 6760 + }, + { + "epoch": 0.44, + "learning_rate": 3.4128210165364837e-06, + "logits/chosen": -2.191530227661133, + "logits/rejected": -2.2053451538085938, + "logps/chosen": -206.8775634765625, + "logps/rejected": -237.47329711914062, + "loss": 1608.8537, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15468339622020721, + "rewards/margins": 0.22072303295135498, + "rewards/rejected": -0.3754064440727234, + "step": 6770 + }, + { + "epoch": 0.44, + "learning_rate": 3.407502958369795e-06, + "logits/chosen": -2.3007140159606934, + "logits/rejected": -2.1566667556762695, + "logps/chosen": -261.19329833984375, + "logps/rejected": -246.59744262695312, + "loss": 1946.2852, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.18745502829551697, + "rewards/margins": 0.1461646854877472, + "rewards/rejected": -0.33361974358558655, + "step": 6780 + }, + { + "epoch": 0.44, + "learning_rate": 3.4021801662231297e-06, + "logits/chosen": -2.338116407394409, + "logits/rejected": -2.188136577606201, + "logps/chosen": -268.2184143066406, + "logps/rejected": -258.36224365234375, + "loss": 1982.3238, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.19341619312763214, + "rewards/margins": 0.12419227510690689, + "rewards/rejected": -0.31760844588279724, + "step": 6790 + }, + { + "epoch": 0.44, + "learning_rate": 3.3968526678627793e-06, + "logits/chosen": -2.2878658771514893, + "logits/rejected": -1.9563350677490234, + "logps/chosen": -279.77392578125, + "logps/rejected": -251.26205444335938, + "loss": 1672.7229, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14743436872959137, + "rewards/margins": 0.15591710805892944, + "rewards/rejected": -0.303351491689682, + "step": 6800 + }, + { + "epoch": 0.44, + "eval_logits/chosen": -2.351219415664673, + "eval_logits/rejected": -2.164029359817505, + "eval_logps/chosen": -250.83648681640625, + "eval_logps/rejected": -244.66224670410156, + "eval_loss": 1816.673095703125, + "eval_rewards/accuracies": 0.652999997138977, + "eval_rewards/chosen": -0.1883152276277542, + "eval_rewards/margins": 0.14218826591968536, + "eval_rewards/rejected": -0.33050352334976196, + "eval_runtime": 724.154, + "eval_samples_per_second": 2.762, + "eval_steps_per_second": 1.381, + "step": 6800 + }, + { + "epoch": 0.45, + "learning_rate": 3.391520491079586e-06, + "logits/chosen": -2.4422621726989746, + "logits/rejected": -2.4687561988830566, + "logps/chosen": -210.08804321289062, + "logps/rejected": -193.15184020996094, + "loss": 1662.1758, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1412539780139923, + "rewards/margins": 0.1359347254037857, + "rewards/rejected": -0.2771887183189392, + "step": 6810 + }, + { + "epoch": 0.45, + "learning_rate": 3.3861836636887936e-06, + "logits/chosen": -2.3429911136627197, + "logits/rejected": -2.1664414405822754, + "logps/chosen": -285.408203125, + "logps/rejected": -253.633056640625, + "loss": 1821.6008, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1858934909105301, + "rewards/margins": 0.1470334827899933, + "rewards/rejected": -0.3329269587993622, + "step": 6820 + }, + { + "epoch": 0.45, + "learning_rate": 3.3808422135299106e-06, + "logits/chosen": -2.3195672035217285, + "logits/rejected": -2.4099817276000977, + "logps/chosen": -303.9296875, + "logps/rejected": -370.9049987792969, + "loss": 1800.8039, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1825186312198639, + "rewards/margins": 0.138997882604599, + "rewards/rejected": -0.3215165138244629, + "step": 6830 + }, + { + "epoch": 0.45, + "learning_rate": 3.375496168466556e-06, + "logits/chosen": -2.50669527053833, + "logits/rejected": -2.0542101860046387, + "logps/chosen": -230.8943328857422, + "logps/rejected": -188.94871520996094, + "loss": 1934.2203, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.15956223011016846, + "rewards/margins": 0.1398100107908249, + "rewards/rejected": -0.29937228560447693, + "step": 6840 + }, + { + "epoch": 0.45, + "learning_rate": 3.3701455563863205e-06, + "logits/chosen": -2.559258222579956, + "logits/rejected": -2.107353687286377, + "logps/chosen": -316.3973083496094, + "logps/rejected": -295.91229248046875, + "loss": 1993.9049, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.24522638320922852, + "rewards/margins": 0.1583399623632431, + "rewards/rejected": -0.4035663604736328, + "step": 6850 + }, + { + "epoch": 0.45, + "learning_rate": 3.3647904052006174e-06, + "logits/chosen": -2.3529694080352783, + "logits/rejected": -2.2934610843658447, + "logps/chosen": -282.9671325683594, + "logps/rejected": -302.84149169921875, + "loss": 1705.6371, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.17668868601322174, + "rewards/margins": 0.16867773234844208, + "rewards/rejected": -0.3453664183616638, + "step": 6860 + }, + { + "epoch": 0.45, + "learning_rate": 3.3594307428445383e-06, + "logits/chosen": -2.5529115200042725, + "logits/rejected": -2.1537489891052246, + "logps/chosen": -345.7641906738281, + "logps/rejected": -332.9287109375, + "loss": 1384.0822, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12513437867164612, + "rewards/margins": 0.1858106553554535, + "rewards/rejected": -0.3109450340270996, + "step": 6870 + }, + { + "epoch": 0.45, + "learning_rate": 3.354066597276707e-06, + "logits/chosen": -2.222578287124634, + "logits/rejected": -2.2239785194396973, + "logps/chosen": -237.1717987060547, + "logps/rejected": -285.36175537109375, + "loss": 2059.5422, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13607537746429443, + "rewards/margins": 0.12829408049583435, + "rewards/rejected": -0.2643694281578064, + "step": 6880 + }, + { + "epoch": 0.45, + "learning_rate": 3.348697996479136e-06, + "logits/chosen": -2.4354445934295654, + "logits/rejected": -2.1811060905456543, + "logps/chosen": -244.497314453125, + "logps/rejected": -204.57057189941406, + "loss": 1864.0285, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1668837070465088, + "rewards/margins": 0.13544951379299164, + "rewards/rejected": -0.30233320593833923, + "step": 6890 + }, + { + "epoch": 0.45, + "learning_rate": 3.3433249684570757e-06, + "logits/chosen": -2.26200270652771, + "logits/rejected": -2.129823684692383, + "logps/chosen": -190.80728149414062, + "logps/rejected": -162.08505249023438, + "loss": 1800.1967, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12977729737758636, + "rewards/margins": 0.1699616014957428, + "rewards/rejected": -0.29973888397216797, + "step": 6900 + }, + { + "epoch": 0.45, + "eval_logits/chosen": -2.350261688232422, + "eval_logits/rejected": -2.163565158843994, + "eval_logps/chosen": -246.64015197753906, + "eval_logps/rejected": -241.98065185546875, + "eval_loss": 1805.9200439453125, + "eval_rewards/accuracies": 0.6514999866485596, + "eval_rewards/chosen": -0.14635208249092102, + "eval_rewards/margins": 0.15733551979064941, + "eval_rewards/rejected": -0.3036876320838928, + "eval_runtime": 723.7743, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.382, + "step": 6900 + }, + { + "epoch": 0.45, + "learning_rate": 3.3379475412388724e-06, + "logits/chosen": -2.432903289794922, + "logits/rejected": -2.272477626800537, + "logps/chosen": -254.492919921875, + "logps/rejected": -242.8998565673828, + "loss": 1273.1773, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.12537913024425507, + "rewards/margins": 0.22105541825294495, + "rewards/rejected": -0.3464345335960388, + "step": 6910 + }, + { + "epoch": 0.45, + "learning_rate": 3.3325657428758207e-06, + "logits/chosen": -2.238603353500366, + "logits/rejected": -2.215906858444214, + "logps/chosen": -252.9639434814453, + "logps/rejected": -273.2824401855469, + "loss": 1748.925, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09288422763347626, + "rewards/margins": 0.19491823017597198, + "rewards/rejected": -0.28780245780944824, + "step": 6920 + }, + { + "epoch": 0.45, + "learning_rate": 3.3271796014420175e-06, + "logits/chosen": -2.3474555015563965, + "logits/rejected": -2.29400372505188, + "logps/chosen": -226.4741668701172, + "logps/rejected": -225.4298095703125, + "loss": 1809.5197, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14023862779140472, + "rewards/margins": 0.16663934290409088, + "rewards/rejected": -0.3068779408931732, + "step": 6930 + }, + { + "epoch": 0.45, + "learning_rate": 3.3217891450342142e-06, + "logits/chosen": -2.341958522796631, + "logits/rejected": -2.0191941261291504, + "logps/chosen": -265.12890625, + "logps/rejected": -217.28939819335938, + "loss": 1436.6966, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06530490517616272, + "rewards/margins": 0.21153318881988525, + "rewards/rejected": -0.2768380641937256, + "step": 6940 + }, + { + "epoch": 0.45, + "learning_rate": 3.3163944017716733e-06, + "logits/chosen": -2.5168886184692383, + "logits/rejected": -2.2140870094299316, + "logps/chosen": -224.3677520751953, + "logps/rejected": -201.74960327148438, + "loss": 2093.9619, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.07996897399425507, + "rewards/margins": 0.10479603707790375, + "rewards/rejected": -0.1847650110721588, + "step": 6950 + }, + { + "epoch": 0.46, + "learning_rate": 3.310995399796017e-06, + "logits/chosen": -2.439115524291992, + "logits/rejected": -2.318527936935425, + "logps/chosen": -282.1032409667969, + "logps/rejected": -285.8451232910156, + "loss": 2300.6125, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.08117020130157471, + "rewards/margins": 0.10543999820947647, + "rewards/rejected": -0.18661019206047058, + "step": 6960 + }, + { + "epoch": 0.46, + "learning_rate": 3.305592167271085e-06, + "logits/chosen": -2.317767858505249, + "logits/rejected": -2.2276875972747803, + "logps/chosen": -202.17050170898438, + "logps/rejected": -208.3937225341797, + "loss": 1673.9172, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.04924345016479492, + "rewards/margins": 0.1713528037071228, + "rewards/rejected": -0.22059623897075653, + "step": 6970 + }, + { + "epoch": 0.46, + "learning_rate": 3.3001847323827846e-06, + "logits/chosen": -2.304652690887451, + "logits/rejected": -2.251988649368286, + "logps/chosen": -272.10894775390625, + "logps/rejected": -285.6690368652344, + "loss": 1587.1257, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.04011273384094238, + "rewards/margins": 0.16978901624679565, + "rewards/rejected": -0.20990173518657684, + "step": 6980 + }, + { + "epoch": 0.46, + "learning_rate": 3.2947731233389447e-06, + "logits/chosen": -2.3865530490875244, + "logits/rejected": -2.018869161605835, + "logps/chosen": -253.6022491455078, + "logps/rejected": -214.45663452148438, + "loss": 1568.9052, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.060955680906772614, + "rewards/margins": 0.16281774640083313, + "rewards/rejected": -0.22377343475818634, + "step": 6990 + }, + { + "epoch": 0.46, + "learning_rate": 3.2893573683691706e-06, + "logits/chosen": -2.2005510330200195, + "logits/rejected": -2.2169203758239746, + "logps/chosen": -209.6483154296875, + "logps/rejected": -207.0091094970703, + "loss": 1767.4076, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.04954850673675537, + "rewards/margins": 0.15256556868553162, + "rewards/rejected": -0.20211410522460938, + "step": 7000 + }, + { + "epoch": 0.46, + "eval_logits/chosen": -2.329406261444092, + "eval_logits/rejected": -2.144054651260376, + "eval_logps/chosen": -238.76791381835938, + "eval_logps/rejected": -232.6849365234375, + "eval_loss": 1820.3804931640625, + "eval_rewards/accuracies": 0.6464999914169312, + "eval_rewards/chosen": -0.06762954592704773, + "eval_rewards/margins": 0.14310090243816376, + "eval_rewards/rejected": -0.21073046326637268, + "eval_runtime": 723.3553, + "eval_samples_per_second": 2.765, + "eval_steps_per_second": 1.382, + "step": 7000 + }, + { + "epoch": 0.46, + "learning_rate": 3.2839374957246915e-06, + "logits/chosen": -2.412297010421753, + "logits/rejected": -2.1327645778656006, + "logps/chosen": -276.3094482421875, + "logps/rejected": -195.72564697265625, + "loss": 2028.4072, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.10219154506921768, + "rewards/margins": 0.10954372584819794, + "rewards/rejected": -0.2117352932691574, + "step": 7010 + }, + { + "epoch": 0.46, + "learning_rate": 3.2785135336782187e-06, + "logits/chosen": -2.2545413970947266, + "logits/rejected": -2.0809073448181152, + "logps/chosen": -245.10043334960938, + "logps/rejected": -285.3205261230469, + "loss": 1585.4265, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.11185015738010406, + "rewards/margins": 0.17907771468162537, + "rewards/rejected": -0.2909278869628906, + "step": 7020 + }, + { + "epoch": 0.46, + "learning_rate": 3.2730855105237952e-06, + "logits/chosen": -2.421980619430542, + "logits/rejected": -2.2304441928863525, + "logps/chosen": -226.10519409179688, + "logps/rejected": -288.8428039550781, + "loss": 1519.8598, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.058287106454372406, + "rewards/margins": 0.18557073175907135, + "rewards/rejected": -0.24385783076286316, + "step": 7030 + }, + { + "epoch": 0.46, + "learning_rate": 3.2676534545766486e-06, + "logits/chosen": -2.2319369316101074, + "logits/rejected": -2.206944465637207, + "logps/chosen": -218.6344451904297, + "logps/rejected": -220.7762908935547, + "loss": 1949.5209, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.06164904311299324, + "rewards/margins": 0.12034694850444794, + "rewards/rejected": -0.18199597299098969, + "step": 7040 + }, + { + "epoch": 0.46, + "learning_rate": 3.262217394173043e-06, + "logits/chosen": -2.341109037399292, + "logits/rejected": -2.059361457824707, + "logps/chosen": -252.3672637939453, + "logps/rejected": -258.5021057128906, + "loss": 1971.7896, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11516623198986053, + "rewards/margins": 0.12217897176742554, + "rewards/rejected": -0.23734521865844727, + "step": 7050 + }, + { + "epoch": 0.46, + "learning_rate": 3.2567773576701333e-06, + "logits/chosen": -2.163533926010132, + "logits/rejected": -1.9948575496673584, + "logps/chosen": -260.88348388671875, + "logps/rejected": -250.1443634033203, + "loss": 1553.0061, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.05126018449664116, + "rewards/margins": 0.1957646906375885, + "rewards/rejected": -0.24702484905719757, + "step": 7060 + }, + { + "epoch": 0.46, + "learning_rate": 3.2513333734458154e-06, + "logits/chosen": -2.377007246017456, + "logits/rejected": -2.2850708961486816, + "logps/chosen": -216.84231567382812, + "logps/rejected": -211.0592803955078, + "loss": 1930.9227, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.10001027584075928, + "rewards/margins": 0.10824619233608246, + "rewards/rejected": -0.20825648307800293, + "step": 7070 + }, + { + "epoch": 0.46, + "learning_rate": 3.245885469898576e-06, + "logits/chosen": -2.271165609359741, + "logits/rejected": -2.0621328353881836, + "logps/chosen": -307.70196533203125, + "logps/rejected": -260.340087890625, + "loss": 1894.34, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0714680626988411, + "rewards/margins": 0.16326221823692322, + "rewards/rejected": -0.2347303181886673, + "step": 7080 + }, + { + "epoch": 0.46, + "learning_rate": 3.2404336754473497e-06, + "logits/chosen": -2.2628307342529297, + "logits/rejected": -2.0071568489074707, + "logps/chosen": -269.6414794921875, + "logps/rejected": -222.27792358398438, + "loss": 1584.2591, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.021345099434256554, + "rewards/margins": 0.17406703531742096, + "rewards/rejected": -0.19541212916374207, + "step": 7090 + }, + { + "epoch": 0.46, + "learning_rate": 3.234978018531367e-06, + "logits/chosen": -2.57427716255188, + "logits/rejected": -2.1523544788360596, + "logps/chosen": -262.66851806640625, + "logps/rejected": -215.177490234375, + "loss": 1597.6004, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.041683003306388855, + "rewards/margins": 0.17308592796325684, + "rewards/rejected": -0.2147689312696457, + "step": 7100 + }, + { + "epoch": 0.46, + "eval_logits/chosen": -2.309113025665283, + "eval_logits/rejected": -2.1259310245513916, + "eval_logps/chosen": -240.7786407470703, + "eval_logps/rejected": -235.21725463867188, + "eval_loss": 1808.856201171875, + "eval_rewards/accuracies": 0.6499999761581421, + "eval_rewards/chosen": -0.08773694932460785, + "eval_rewards/margins": 0.14831653237342834, + "eval_rewards/rejected": -0.2360534816980362, + "eval_runtime": 724.9942, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.379, + "step": 7100 + }, + { + "epoch": 0.47, + "learning_rate": 3.229518527610006e-06, + "logits/chosen": -2.396723747253418, + "logits/rejected": -2.1092593669891357, + "logps/chosen": -302.719482421875, + "logps/rejected": -269.29229736328125, + "loss": 1988.2623, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09929381310939789, + "rewards/margins": 0.11748860031366348, + "rewards/rejected": -0.21678242087364197, + "step": 7110 + }, + { + "epoch": 0.47, + "learning_rate": 3.2240552311626465e-06, + "logits/chosen": -2.3822884559631348, + "logits/rejected": -2.1514086723327637, + "logps/chosen": -248.64242553710938, + "logps/rejected": -245.8981475830078, + "loss": 2073.4006, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.07428698241710663, + "rewards/margins": 0.12200691550970078, + "rewards/rejected": -0.196293905377388, + "step": 7120 + }, + { + "epoch": 0.47, + "learning_rate": 3.2185881576885193e-06, + "logits/chosen": -2.4048800468444824, + "logits/rejected": -2.0579912662506104, + "logps/chosen": -225.2511749267578, + "logps/rejected": -197.3291473388672, + "loss": 1877.8533, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1352832019329071, + "rewards/margins": 0.10745497047901154, + "rewards/rejected": -0.24273820221424103, + "step": 7130 + }, + { + "epoch": 0.47, + "learning_rate": 3.213117335706557e-06, + "logits/chosen": -2.283663511276245, + "logits/rejected": -2.39902925491333, + "logps/chosen": -267.2200012207031, + "logps/rejected": -287.21697998046875, + "loss": 2213.0275, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11175023019313812, + "rewards/margins": 0.10790754854679108, + "rewards/rejected": -0.2196577787399292, + "step": 7140 + }, + { + "epoch": 0.47, + "learning_rate": 3.2076427937552473e-06, + "logits/chosen": -2.3167736530303955, + "logits/rejected": -2.0965049266815186, + "logps/chosen": -254.1221160888672, + "logps/rejected": -253.3069610595703, + "loss": 1572.583, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.0863238126039505, + "rewards/margins": 0.18308484554290771, + "rewards/rejected": -0.2694086730480194, + "step": 7150 + }, + { + "epoch": 0.47, + "learning_rate": 3.2021645603924827e-06, + "logits/chosen": -2.1734702587127686, + "logits/rejected": -2.078382968902588, + "logps/chosen": -147.8426513671875, + "logps/rejected": -173.65969848632812, + "loss": 1859.6615, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.0938853994011879, + "rewards/margins": 0.16452589631080627, + "rewards/rejected": -0.2584112882614136, + "step": 7160 + }, + { + "epoch": 0.47, + "learning_rate": 3.196682664195412e-06, + "logits/chosen": -2.296813488006592, + "logits/rejected": -2.0265824794769287, + "logps/chosen": -212.2986297607422, + "logps/rejected": -186.99539184570312, + "loss": 2164.1652, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.08250459283590317, + "rewards/margins": 0.09587766975164413, + "rewards/rejected": -0.1783822774887085, + "step": 7170 + }, + { + "epoch": 0.47, + "learning_rate": 3.191197133760291e-06, + "logits/chosen": -2.542553424835205, + "logits/rejected": -2.2270288467407227, + "logps/chosen": -267.11199951171875, + "logps/rejected": -215.85452270507812, + "loss": 1465.4353, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.0029144559521228075, + "rewards/margins": 0.22991275787353516, + "rewards/rejected": -0.22699828445911407, + "step": 7180 + }, + { + "epoch": 0.47, + "learning_rate": 3.185707997702334e-06, + "logits/chosen": -2.202695369720459, + "logits/rejected": -2.0384864807128906, + "logps/chosen": -249.30551147460938, + "logps/rejected": -224.59207153320312, + "loss": 2006.3016, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08540628850460052, + "rewards/margins": 0.13138136267662048, + "rewards/rejected": -0.216787651181221, + "step": 7190 + }, + { + "epoch": 0.47, + "learning_rate": 3.1802152846555624e-06, + "logits/chosen": -2.2406742572784424, + "logits/rejected": -2.2176101207733154, + "logps/chosen": -231.08535766601562, + "logps/rejected": -232.2159881591797, + "loss": 1694.7447, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.0684235543012619, + "rewards/margins": 0.16649962961673737, + "rewards/rejected": -0.23492319881916046, + "step": 7200 + }, + { + "epoch": 0.47, + "eval_logits/chosen": -2.3213398456573486, + "eval_logits/rejected": -2.136472225189209, + "eval_logps/chosen": -238.93997192382812, + "eval_logps/rejected": -232.4331512451172, + "eval_loss": 1807.7432861328125, + "eval_rewards/accuracies": 0.6485000252723694, + "eval_rewards/chosen": -0.06935019791126251, + "eval_rewards/margins": 0.13886241614818573, + "eval_rewards/rejected": -0.20821261405944824, + "eval_runtime": 722.9812, + "eval_samples_per_second": 2.766, + "eval_steps_per_second": 1.383, + "step": 7200 + }, + { + "epoch": 0.47, + "learning_rate": 3.174719023272659e-06, + "logits/chosen": -2.395876169204712, + "logits/rejected": -2.441063404083252, + "logps/chosen": -220.5709686279297, + "logps/rejected": -281.8936462402344, + "loss": 1723.2258, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.06876830756664276, + "rewards/margins": 0.1497349888086319, + "rewards/rejected": -0.21850331127643585, + "step": 7210 + }, + { + "epoch": 0.47, + "learning_rate": 3.169219242224816e-06, + "logits/chosen": -2.366623878479004, + "logits/rejected": -2.1990063190460205, + "logps/chosen": -251.0176544189453, + "logps/rejected": -256.62091064453125, + "loss": 1895.4932, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11090411245822906, + "rewards/margins": 0.11698516458272934, + "rewards/rejected": -0.2278892993927002, + "step": 7220 + }, + { + "epoch": 0.47, + "learning_rate": 3.1637159702015837e-06, + "logits/chosen": -2.3689377307891846, + "logits/rejected": -2.057278871536255, + "logps/chosen": -212.3173370361328, + "logps/rejected": -213.333984375, + "loss": 1277.8221, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06425191462039948, + "rewards/margins": 0.20339460670948029, + "rewards/rejected": -0.26764652132987976, + "step": 7230 + }, + { + "epoch": 0.47, + "learning_rate": 3.1582092359107263e-06, + "logits/chosen": -2.417332410812378, + "logits/rejected": -2.196626901626587, + "logps/chosen": -285.0397033691406, + "logps/rejected": -255.1283416748047, + "loss": 1720.6148, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06761996448040009, + "rewards/margins": 0.1591935008764267, + "rewards/rejected": -0.22681346535682678, + "step": 7240 + }, + { + "epoch": 0.47, + "learning_rate": 3.152699068078067e-06, + "logits/chosen": -2.2219226360321045, + "logits/rejected": -2.021028518676758, + "logps/chosen": -291.9150085449219, + "logps/rejected": -286.8653564453125, + "loss": 1690.582, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.11624713987112045, + "rewards/margins": 0.15491464734077454, + "rewards/rejected": -0.2711617350578308, + "step": 7250 + }, + { + "epoch": 0.48, + "learning_rate": 3.1471854954473415e-06, + "logits/chosen": -2.366694688796997, + "logits/rejected": -2.4141058921813965, + "logps/chosen": -253.8938446044922, + "logps/rejected": -267.23431396484375, + "loss": 1545.9211, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.03127194195985794, + "rewards/margins": 0.17108400166034698, + "rewards/rejected": -0.20235593616962433, + "step": 7260 + }, + { + "epoch": 0.48, + "learning_rate": 3.1416685467800436e-06, + "logits/chosen": -2.163619041442871, + "logits/rejected": -2.132086992263794, + "logps/chosen": -197.5450897216797, + "logps/rejected": -190.2638702392578, + "loss": 1734.2273, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11962287127971649, + "rewards/margins": 0.15248456597328186, + "rewards/rejected": -0.27210742235183716, + "step": 7270 + }, + { + "epoch": 0.48, + "learning_rate": 3.1361482508552803e-06, + "logits/chosen": -2.3619093894958496, + "logits/rejected": -1.863598108291626, + "logps/chosen": -255.34628295898438, + "logps/rejected": -225.1408233642578, + "loss": 1790.4477, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13395367562770844, + "rewards/margins": 0.1318083107471466, + "rewards/rejected": -0.26576200127601624, + "step": 7280 + }, + { + "epoch": 0.48, + "learning_rate": 3.1306246364696198e-06, + "logits/chosen": -2.48795747756958, + "logits/rejected": -2.2847416400909424, + "logps/chosen": -265.2281799316406, + "logps/rejected": -259.6405029296875, + "loss": 1982.2109, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12753859162330627, + "rewards/margins": 0.11917465925216675, + "rewards/rejected": -0.24671323597431183, + "step": 7290 + }, + { + "epoch": 0.48, + "learning_rate": 3.1250977324369413e-06, + "logits/chosen": -2.272779941558838, + "logits/rejected": -2.19761323928833, + "logps/chosen": -170.6405029296875, + "logps/rejected": -193.0902557373047, + "loss": 1729.9865, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11688047647476196, + "rewards/margins": 0.15737764537334442, + "rewards/rejected": -0.2742581069469452, + "step": 7300 + }, + { + "epoch": 0.48, + "eval_logits/chosen": -2.3204145431518555, + "eval_logits/rejected": -2.1347432136535645, + "eval_logps/chosen": -244.92276000976562, + "eval_logps/rejected": -239.7329864501953, + "eval_loss": 1804.16455078125, + "eval_rewards/accuracies": 0.6575000286102295, + "eval_rewards/chosen": -0.12917816638946533, + "eval_rewards/margins": 0.1520327478647232, + "eval_rewards/rejected": -0.28121089935302734, + "eval_runtime": 724.3424, + "eval_samples_per_second": 2.761, + "eval_steps_per_second": 1.381, + "step": 7300 + }, + { + "epoch": 0.48, + "learning_rate": 3.1195675675882825e-06, + "logits/chosen": -2.229360580444336, + "logits/rejected": -2.124525547027588, + "logps/chosen": -254.34945678710938, + "logps/rejected": -234.5399932861328, + "loss": 2264.6254, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1915845423936844, + "rewards/margins": 0.10795316845178604, + "rewards/rejected": -0.29953768849372864, + "step": 7310 + }, + { + "epoch": 0.48, + "learning_rate": 3.1140341707716926e-06, + "logits/chosen": -2.1734554767608643, + "logits/rejected": -1.928881287574768, + "logps/chosen": -213.058837890625, + "logps/rejected": -185.9547882080078, + "loss": 1667.8791, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.17251141369342804, + "rewards/margins": 0.15912136435508728, + "rewards/rejected": -0.3316327929496765, + "step": 7320 + }, + { + "epoch": 0.48, + "learning_rate": 3.1084975708520803e-06, + "logits/chosen": -2.422889232635498, + "logits/rejected": -2.0278878211975098, + "logps/chosen": -277.5769958496094, + "logps/rejected": -226.4959259033203, + "loss": 1500.3472, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15853142738342285, + "rewards/margins": 0.1966545730829239, + "rewards/rejected": -0.35518598556518555, + "step": 7330 + }, + { + "epoch": 0.48, + "learning_rate": 3.1029577967110625e-06, + "logits/chosen": -2.4203174114227295, + "logits/rejected": -2.207869291305542, + "logps/chosen": -232.2501983642578, + "logps/rejected": -193.13587951660156, + "loss": 2062.1725, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1585586816072464, + "rewards/margins": 0.09944029152393341, + "rewards/rejected": -0.2579990029335022, + "step": 7340 + }, + { + "epoch": 0.48, + "learning_rate": 3.097414877246814e-06, + "logits/chosen": -2.263326406478882, + "logits/rejected": -2.018284320831299, + "logps/chosen": -212.86083984375, + "logps/rejected": -198.14041137695312, + "loss": 1925.0178, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14217372238636017, + "rewards/margins": 0.14792440831661224, + "rewards/rejected": -0.2900981307029724, + "step": 7350 + }, + { + "epoch": 0.48, + "learning_rate": 3.0918688413739197e-06, + "logits/chosen": -2.3319170475006104, + "logits/rejected": -2.0233492851257324, + "logps/chosen": -243.54483032226562, + "logps/rejected": -193.8897247314453, + "loss": 1834.7447, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.10861881077289581, + "rewards/margins": 0.14617590606212616, + "rewards/rejected": -0.254794716835022, + "step": 7360 + }, + { + "epoch": 0.48, + "learning_rate": 3.0863197180232178e-06, + "logits/chosen": -2.3774733543395996, + "logits/rejected": -2.0132126808166504, + "logps/chosen": -213.40408325195312, + "logps/rejected": -212.970703125, + "loss": 2035.8984, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.13659898936748505, + "rewards/margins": 0.11787278950214386, + "rewards/rejected": -0.2544717490673065, + "step": 7370 + }, + { + "epoch": 0.48, + "learning_rate": 3.0807675361416554e-06, + "logits/chosen": -2.2831597328186035, + "logits/rejected": -2.0448970794677734, + "logps/chosen": -198.5185546875, + "logps/rejected": -134.53500366210938, + "loss": 1672.6324, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09638302028179169, + "rewards/margins": 0.15050837397575378, + "rewards/rejected": -0.24689140915870667, + "step": 7380 + }, + { + "epoch": 0.48, + "learning_rate": 3.0752123246921327e-06, + "logits/chosen": -2.4044673442840576, + "logits/rejected": -2.1336283683776855, + "logps/chosen": -296.2977600097656, + "logps/rejected": -243.39669799804688, + "loss": 1830.0312, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.18384496867656708, + "rewards/margins": 0.14030399918556213, + "rewards/rejected": -0.324148952960968, + "step": 7390 + }, + { + "epoch": 0.48, + "learning_rate": 3.069654112653353e-06, + "logits/chosen": -2.4212207794189453, + "logits/rejected": -2.220515251159668, + "logps/chosen": -235.97140502929688, + "logps/rejected": -212.03701782226562, + "loss": 2149.3504, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.19487205147743225, + "rewards/margins": 0.08617673814296722, + "rewards/rejected": -0.28104880452156067, + "step": 7400 + }, + { + "epoch": 0.48, + "eval_logits/chosen": -2.3111214637756348, + "eval_logits/rejected": -2.126542329788208, + "eval_logps/chosen": -248.8470001220703, + "eval_logps/rejected": -243.38546752929688, + "eval_loss": 1797.1915283203125, + "eval_rewards/accuracies": 0.6520000100135803, + "eval_rewards/chosen": -0.16842052340507507, + "eval_rewards/margins": 0.14931510388851166, + "eval_rewards/rejected": -0.31773561239242554, + "eval_runtime": 724.8176, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 7400 + }, + { + "epoch": 0.48, + "learning_rate": 3.064092929019673e-06, + "logits/chosen": -2.301642656326294, + "logits/rejected": -2.334155559539795, + "logps/chosen": -275.03240966796875, + "logps/rejected": -306.5906066894531, + "loss": 1980.4135, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1748892217874527, + "rewards/margins": 0.11683398485183716, + "rewards/rejected": -0.29172322154045105, + "step": 7410 + }, + { + "epoch": 0.49, + "learning_rate": 3.058528802800952e-06, + "logits/chosen": -2.3407881259918213, + "logits/rejected": -2.1059513092041016, + "logps/chosen": -308.2147216796875, + "logps/rejected": -286.90826416015625, + "loss": 1935.9912, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.15580996870994568, + "rewards/margins": 0.15266405045986176, + "rewards/rejected": -0.30847403407096863, + "step": 7420 + }, + { + "epoch": 0.49, + "learning_rate": 3.052961763022397e-06, + "logits/chosen": -2.486992120742798, + "logits/rejected": -2.16389799118042, + "logps/chosen": -200.91552734375, + "logps/rejected": -179.10459899902344, + "loss": 1560.0854, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1803436130285263, + "rewards/margins": 0.16912725567817688, + "rewards/rejected": -0.3494708836078644, + "step": 7430 + }, + { + "epoch": 0.49, + "learning_rate": 3.047391838724415e-06, + "logits/chosen": -2.4561057090759277, + "logits/rejected": -1.99441659450531, + "logps/chosen": -252.029296875, + "logps/rejected": -252.39501953125, + "loss": 1582.4479, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.16510634124279022, + "rewards/margins": 0.16432276368141174, + "rewards/rejected": -0.32942909002304077, + "step": 7440 + }, + { + "epoch": 0.49, + "learning_rate": 3.0418190589624587e-06, + "logits/chosen": -2.389727830886841, + "logits/rejected": -2.125429391860962, + "logps/chosen": -192.25135803222656, + "logps/rejected": -213.27542114257812, + "loss": 1738.7389, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14629261195659637, + "rewards/margins": 0.12671688199043274, + "rewards/rejected": -0.2730094790458679, + "step": 7450 + }, + { + "epoch": 0.49, + "learning_rate": 3.0362434528068784e-06, + "logits/chosen": -2.372622013092041, + "logits/rejected": -1.9482421875, + "logps/chosen": -288.8235778808594, + "logps/rejected": -217.744140625, + "loss": 1948.5447, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1950044482946396, + "rewards/margins": 0.1441466361284256, + "rewards/rejected": -0.3391510844230652, + "step": 7460 + }, + { + "epoch": 0.49, + "learning_rate": 3.0306650493427657e-06, + "logits/chosen": -2.264923095703125, + "logits/rejected": -2.1652779579162598, + "logps/chosen": -249.77615356445312, + "logps/rejected": -257.1854553222656, + "loss": 1638.9396, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1815689206123352, + "rewards/margins": 0.16371271014213562, + "rewards/rejected": -0.3452816605567932, + "step": 7470 + }, + { + "epoch": 0.49, + "learning_rate": 3.0250838776698077e-06, + "logits/chosen": -2.133946418762207, + "logits/rejected": -2.1851963996887207, + "logps/chosen": -204.72146606445312, + "logps/rejected": -217.3024444580078, + "loss": 1626.3103, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.2080218344926834, + "rewards/margins": 0.15327318012714386, + "rewards/rejected": -0.3612949848175049, + "step": 7480 + }, + { + "epoch": 0.49, + "learning_rate": 3.0194999669021275e-06, + "logits/chosen": -2.1351277828216553, + "logits/rejected": -1.8072388172149658, + "logps/chosen": -243.4135284423828, + "logps/rejected": -218.5195770263672, + "loss": 1347.7062, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16545066237449646, + "rewards/margins": 0.22158046066761017, + "rewards/rejected": -0.3870311379432678, + "step": 7490 + }, + { + "epoch": 0.49, + "learning_rate": 3.0139133461681403e-06, + "logits/chosen": -2.2730579376220703, + "logits/rejected": -2.1299405097961426, + "logps/chosen": -282.8431091308594, + "logps/rejected": -246.6148681640625, + "loss": 1329.7865, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.18448880314826965, + "rewards/margins": 0.19911891222000122, + "rewards/rejected": -0.3836076855659485, + "step": 7500 + }, + { + "epoch": 0.49, + "eval_logits/chosen": -2.3219900131225586, + "eval_logits/rejected": -2.136725664138794, + "eval_logps/chosen": -249.3482208251953, + "eval_logps/rejected": -244.99862670898438, + "eval_loss": 1795.6614990234375, + "eval_rewards/accuracies": 0.6585000157356262, + "eval_rewards/chosen": -0.17343279719352722, + "eval_rewards/margins": 0.16043433547019958, + "eval_rewards/rejected": -0.3338671326637268, + "eval_runtime": 725.297, + "eval_samples_per_second": 2.757, + "eval_steps_per_second": 1.379, + "step": 7500 + }, + { + "epoch": 0.49, + "learning_rate": 3.0083240446103965e-06, + "logits/chosen": -2.050713062286377, + "logits/rejected": -2.0219483375549316, + "logps/chosen": -202.81639099121094, + "logps/rejected": -226.4502410888672, + "loss": 1777.7549, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.18781790137290955, + "rewards/margins": 0.1596471667289734, + "rewards/rejected": -0.3474650979042053, + "step": 7510 + }, + { + "epoch": 0.49, + "learning_rate": 3.0027320913854306e-06, + "logits/chosen": -2.513092517852783, + "logits/rejected": -2.2261266708374023, + "logps/chosen": -304.5718994140625, + "logps/rejected": -266.6675720214844, + "loss": 1646.9309, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.12110300362110138, + "rewards/margins": 0.2480504810810089, + "rewards/rejected": -0.3691534399986267, + "step": 7520 + }, + { + "epoch": 0.49, + "learning_rate": 2.997137515663609e-06, + "logits/chosen": -2.2597782611846924, + "logits/rejected": -2.1838889122009277, + "logps/chosen": -236.5629425048828, + "logps/rejected": -221.2807159423828, + "loss": 1442.7711, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1118551641702652, + "rewards/margins": 0.22416965663433075, + "rewards/rejected": -0.33602482080459595, + "step": 7530 + }, + { + "epoch": 0.49, + "learning_rate": 2.991540346628981e-06, + "logits/chosen": -2.374667167663574, + "logits/rejected": -2.191114664077759, + "logps/chosen": -263.5557861328125, + "logps/rejected": -251.18161010742188, + "loss": 1976.7508, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.23955456912517548, + "rewards/margins": 0.1373405158519745, + "rewards/rejected": -0.37689509987831116, + "step": 7540 + }, + { + "epoch": 0.49, + "learning_rate": 2.985940613479121e-06, + "logits/chosen": -2.4493889808654785, + "logits/rejected": -2.342282772064209, + "logps/chosen": -312.0015563964844, + "logps/rejected": -273.027099609375, + "loss": 1377.7288, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.18151938915252686, + "rewards/margins": 0.20397362112998962, + "rewards/rejected": -0.38549304008483887, + "step": 7550 + }, + { + "epoch": 0.49, + "learning_rate": 2.980338345424981e-06, + "logits/chosen": -2.3101601600646973, + "logits/rejected": -2.007136821746826, + "logps/chosen": -268.2823791503906, + "logps/rejected": -233.455810546875, + "loss": 1776.1154, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.19062550365924835, + "rewards/margins": 0.13450220227241516, + "rewards/rejected": -0.3251277208328247, + "step": 7560 + }, + { + "epoch": 0.5, + "learning_rate": 2.974733571690735e-06, + "logits/chosen": -2.396482467651367, + "logits/rejected": -2.1252634525299072, + "logps/chosen": -260.60748291015625, + "logps/rejected": -213.5504150390625, + "loss": 2111.8732, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2320062220096588, + "rewards/margins": 0.1223546713590622, + "rewards/rejected": -0.3543609082698822, + "step": 7570 + }, + { + "epoch": 0.5, + "learning_rate": 2.9691263215136274e-06, + "logits/chosen": -2.3606293201446533, + "logits/rejected": -2.3490958213806152, + "logps/chosen": -282.8438415527344, + "logps/rejected": -268.9943542480469, + "loss": 1524.6323, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1584346890449524, + "rewards/margins": 0.17805036902427673, + "rewards/rejected": -0.3364850878715515, + "step": 7580 + }, + { + "epoch": 0.5, + "learning_rate": 2.963516624143823e-06, + "logits/chosen": -2.266894578933716, + "logits/rejected": -2.1448280811309814, + "logps/chosen": -236.0391387939453, + "logps/rejected": -210.7563934326172, + "loss": 2044.7648, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.23472432792186737, + "rewards/margins": 0.12248021364212036, + "rewards/rejected": -0.35720449686050415, + "step": 7590 + }, + { + "epoch": 0.5, + "learning_rate": 2.9579045088442504e-06, + "logits/chosen": -2.1435303688049316, + "logits/rejected": -2.192343235015869, + "logps/chosen": -202.0614776611328, + "logps/rejected": -239.5817413330078, + "loss": 1424.3902, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.13556505739688873, + "rewards/margins": 0.17771312594413757, + "rewards/rejected": -0.3132781684398651, + "step": 7600 + }, + { + "epoch": 0.5, + "eval_logits/chosen": -2.310932159423828, + "eval_logits/rejected": -2.127431631088257, + "eval_logps/chosen": -245.70091247558594, + "eval_logps/rejected": -242.0367431640625, + "eval_loss": 1812.0533447265625, + "eval_rewards/accuracies": 0.656499981880188, + "eval_rewards/chosen": -0.13695943355560303, + "eval_rewards/margins": 0.1672891527414322, + "eval_rewards/rejected": -0.3042486011981964, + "eval_runtime": 724.3066, + "eval_samples_per_second": 2.761, + "eval_steps_per_second": 1.381, + "step": 7600 + }, + { + "epoch": 0.5, + "learning_rate": 2.9522900048904534e-06, + "logits/chosen": -2.2314212322235107, + "logits/rejected": -2.1451785564422607, + "logps/chosen": -259.7351379394531, + "logps/rejected": -238.1786651611328, + "loss": 2245.4512, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1999697983264923, + "rewards/margins": 0.0846501812338829, + "rewards/rejected": -0.2846199870109558, + "step": 7610 + }, + { + "epoch": 0.5, + "learning_rate": 2.9466731415704343e-06, + "logits/chosen": -2.3211567401885986, + "logits/rejected": -2.1959145069122314, + "logps/chosen": -236.20181274414062, + "logps/rejected": -251.24313354492188, + "loss": 1601.2445, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12415148317813873, + "rewards/margins": 0.19408674538135529, + "rewards/rejected": -0.31823819875717163, + "step": 7620 + }, + { + "epoch": 0.5, + "learning_rate": 2.941053948184503e-06, + "logits/chosen": -2.3644680976867676, + "logits/rejected": -2.2212233543395996, + "logps/chosen": -288.4496154785156, + "logps/rejected": -269.64007568359375, + "loss": 1897.9051, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08510540425777435, + "rewards/margins": 0.16102668642997742, + "rewards/rejected": -0.24613209068775177, + "step": 7630 + }, + { + "epoch": 0.5, + "learning_rate": 2.935432454045125e-06, + "logits/chosen": -2.1752820014953613, + "logits/rejected": -2.2406585216522217, + "logps/chosen": -248.2934112548828, + "logps/rejected": -237.24526977539062, + "loss": 2031.8443, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16329045593738556, + "rewards/margins": 0.10159357637166977, + "rewards/rejected": -0.2648840546607971, + "step": 7640 + }, + { + "epoch": 0.5, + "learning_rate": 2.929808688476768e-06, + "logits/chosen": -2.3958911895751953, + "logits/rejected": -2.271388292312622, + "logps/chosen": -252.97970581054688, + "logps/rejected": -251.6690216064453, + "loss": 1745.6922, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1361730545759201, + "rewards/margins": 0.1629178375005722, + "rewards/rejected": -0.2990909218788147, + "step": 7650 + }, + { + "epoch": 0.5, + "learning_rate": 2.924182680815748e-06, + "logits/chosen": -2.317483425140381, + "logits/rejected": -2.247915267944336, + "logps/chosen": -245.28451538085938, + "logps/rejected": -248.97830200195312, + "loss": 1318.8134, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10653947293758392, + "rewards/margins": 0.26401370763778687, + "rewards/rejected": -0.370553195476532, + "step": 7660 + }, + { + "epoch": 0.5, + "learning_rate": 2.9185544604100765e-06, + "logits/chosen": -2.10002064704895, + "logits/rejected": -2.007351875305176, + "logps/chosen": -217.10092163085938, + "logps/rejected": -224.46530151367188, + "loss": 2086.9361, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2119079828262329, + "rewards/margins": 0.11619792133569717, + "rewards/rejected": -0.3281059265136719, + "step": 7670 + }, + { + "epoch": 0.5, + "learning_rate": 2.9129240566193083e-06, + "logits/chosen": -2.4031784534454346, + "logits/rejected": -2.086520195007324, + "logps/chosen": -220.94961547851562, + "logps/rejected": -224.2306671142578, + "loss": 1544.9651, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1886926144361496, + "rewards/margins": 0.16569046676158905, + "rewards/rejected": -0.35438311100006104, + "step": 7680 + }, + { + "epoch": 0.5, + "learning_rate": 2.9072914988143874e-06, + "logits/chosen": -2.161214590072632, + "logits/rejected": -2.080164909362793, + "logps/chosen": -217.6607208251953, + "logps/rejected": -232.4710235595703, + "loss": 1151.4793, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.16230908036231995, + "rewards/margins": 0.2328173667192459, + "rewards/rejected": -0.3951264023780823, + "step": 7690 + }, + { + "epoch": 0.5, + "learning_rate": 2.9016568163774956e-06, + "logits/chosen": -2.3935441970825195, + "logits/rejected": -2.1594557762145996, + "logps/chosen": -183.3972625732422, + "logps/rejected": -164.7067108154297, + "loss": 1652.7855, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11167971789836884, + "rewards/margins": 0.15362325310707092, + "rewards/rejected": -0.26530295610427856, + "step": 7700 + }, + { + "epoch": 0.5, + "eval_logits/chosen": -2.3180606365203857, + "eval_logits/rejected": -2.1330764293670654, + "eval_logps/chosen": -245.59124755859375, + "eval_logps/rejected": -241.61257934570312, + "eval_loss": 1805.7569580078125, + "eval_rewards/accuracies": 0.6524999737739563, + "eval_rewards/chosen": -0.13586299121379852, + "eval_rewards/margins": 0.16414377093315125, + "eval_rewards/rejected": -0.30000677704811096, + "eval_runtime": 724.718, + "eval_samples_per_second": 2.76, + "eval_steps_per_second": 1.38, + "step": 7700 + }, + { + "epoch": 0.5, + "learning_rate": 2.8960200387018942e-06, + "logits/chosen": -2.148181200027466, + "logits/rejected": -2.120988368988037, + "logps/chosen": -326.7958984375, + "logps/rejected": -287.37103271484375, + "loss": 2206.0355, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.19062894582748413, + "rewards/margins": 0.09409534931182861, + "rewards/rejected": -0.28472429513931274, + "step": 7710 + }, + { + "epoch": 0.51, + "learning_rate": 2.8903811951917792e-06, + "logits/chosen": -2.3121659755706787, + "logits/rejected": -2.1631019115448, + "logps/chosen": -208.6977081298828, + "logps/rejected": -175.44943237304688, + "loss": 1798.4217, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.104180708527565, + "rewards/margins": 0.13893906772136688, + "rewards/rejected": -0.2431197613477707, + "step": 7720 + }, + { + "epoch": 0.51, + "learning_rate": 2.88474031526212e-06, + "logits/chosen": -2.2695491313934326, + "logits/rejected": -2.2421715259552, + "logps/chosen": -213.50888061523438, + "logps/rejected": -238.98361206054688, + "loss": 1972.915, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1066913828253746, + "rewards/margins": 0.12046156078577042, + "rewards/rejected": -0.22715294361114502, + "step": 7730 + }, + { + "epoch": 0.51, + "learning_rate": 2.879097428338509e-06, + "logits/chosen": -2.263644218444824, + "logits/rejected": -1.9583408832550049, + "logps/chosen": -226.1009521484375, + "logps/rejected": -218.6776885986328, + "loss": 1923.6541, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.10205962508916855, + "rewards/margins": 0.1490507423877716, + "rewards/rejected": -0.25111037492752075, + "step": 7740 + }, + { + "epoch": 0.51, + "learning_rate": 2.8734525638570094e-06, + "logits/chosen": -2.2643754482269287, + "logits/rejected": -2.1896746158599854, + "logps/chosen": -241.3166046142578, + "logps/rejected": -246.44775390625, + "loss": 1724.5227, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07122374325990677, + "rewards/margins": 0.17504428327083588, + "rewards/rejected": -0.24626803398132324, + "step": 7750 + }, + { + "epoch": 0.51, + "learning_rate": 2.8678057512639982e-06, + "logits/chosen": -2.2165639400482178, + "logits/rejected": -2.12363862991333, + "logps/chosen": -291.8692626953125, + "logps/rejected": -293.48944091796875, + "loss": 1396.4198, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.05802815407514572, + "rewards/margins": 0.23679597675800323, + "rewards/rejected": -0.29482412338256836, + "step": 7760 + }, + { + "epoch": 0.51, + "learning_rate": 2.8621570200160172e-06, + "logits/chosen": -2.1056911945343018, + "logits/rejected": -2.0133914947509766, + "logps/chosen": -175.77249145507812, + "logps/rejected": -190.22726440429688, + "loss": 1617.67, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.07203412801027298, + "rewards/margins": 0.22632765769958496, + "rewards/rejected": -0.29836180806159973, + "step": 7770 + }, + { + "epoch": 0.51, + "learning_rate": 2.856506399579615e-06, + "logits/chosen": -2.4490838050842285, + "logits/rejected": -2.0760135650634766, + "logps/chosen": -235.693603515625, + "logps/rejected": -228.88259887695312, + "loss": 2112.6408, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1733010709285736, + "rewards/margins": 0.1267828792333603, + "rewards/rejected": -0.3000839650630951, + "step": 7780 + }, + { + "epoch": 0.51, + "learning_rate": 2.8508539194311964e-06, + "logits/chosen": -2.3463687896728516, + "logits/rejected": -2.335172414779663, + "logps/chosen": -268.44915771484375, + "logps/rejected": -295.9856872558594, + "loss": 1723.3611, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1471569985151291, + "rewards/margins": 0.14706538617610931, + "rewards/rejected": -0.294222354888916, + "step": 7790 + }, + { + "epoch": 0.51, + "learning_rate": 2.8451996090568656e-06, + "logits/chosen": -2.2608280181884766, + "logits/rejected": -2.14410400390625, + "logps/chosen": -199.97055053710938, + "logps/rejected": -201.8147735595703, + "loss": 1540.4484, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13830958306789398, + "rewards/margins": 0.19805839657783508, + "rewards/rejected": -0.33636802434921265, + "step": 7800 + }, + { + "epoch": 0.51, + "eval_logits/chosen": -2.299730062484741, + "eval_logits/rejected": -2.1168112754821777, + "eval_logps/chosen": -247.72259521484375, + "eval_logps/rejected": -242.64073181152344, + "eval_loss": 1808.3577880859375, + "eval_rewards/accuracies": 0.656000018119812, + "eval_rewards/chosen": -0.15717624127864838, + "eval_rewards/margins": 0.15311181545257568, + "eval_rewards/rejected": -0.31028807163238525, + "eval_runtime": 724.637, + "eval_samples_per_second": 2.76, + "eval_steps_per_second": 1.38, + "step": 7800 + }, + { + "epoch": 0.51, + "learning_rate": 2.839543497952276e-06, + "logits/chosen": -2.1900064945220947, + "logits/rejected": -2.298042058944702, + "logps/chosen": -201.76132202148438, + "logps/rejected": -209.26272583007812, + "loss": 1763.2158, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.17151224613189697, + "rewards/margins": 0.16205264627933502, + "rewards/rejected": -0.3335648775100708, + "step": 7810 + }, + { + "epoch": 0.51, + "learning_rate": 2.833885615622474e-06, + "logits/chosen": -2.23939847946167, + "logits/rejected": -2.090911626815796, + "logps/chosen": -228.0774383544922, + "logps/rejected": -247.0799560546875, + "loss": 2073.5559, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.2550160884857178, + "rewards/margins": 0.09527119994163513, + "rewards/rejected": -0.3502873182296753, + "step": 7820 + }, + { + "epoch": 0.51, + "learning_rate": 2.8282259915817454e-06, + "logits/chosen": -1.9413120746612549, + "logits/rejected": -2.143981456756592, + "logps/chosen": -158.98806762695312, + "logps/rejected": -215.71817016601562, + "loss": 1776.5148, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.18043342232704163, + "rewards/margins": 0.16518518328666687, + "rewards/rejected": -0.3456185758113861, + "step": 7830 + }, + { + "epoch": 0.51, + "learning_rate": 2.8225646553534614e-06, + "logits/chosen": -2.1059117317199707, + "logits/rejected": -1.999498963356018, + "logps/chosen": -213.69155883789062, + "logps/rejected": -221.0135040283203, + "loss": 2001.6479, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.14375567436218262, + "rewards/margins": 0.10613244771957397, + "rewards/rejected": -0.24988813698291779, + "step": 7840 + }, + { + "epoch": 0.51, + "learning_rate": 2.8169016364699255e-06, + "logits/chosen": -2.2967660427093506, + "logits/rejected": -2.025442123413086, + "logps/chosen": -234.79638671875, + "logps/rejected": -250.521240234375, + "loss": 1723.0146, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.22908520698547363, + "rewards/margins": 0.1397470086812973, + "rewards/rejected": -0.36883223056793213, + "step": 7850 + }, + { + "epoch": 0.51, + "learning_rate": 2.811236964472217e-06, + "logits/chosen": -2.4080090522766113, + "logits/rejected": -2.0328783988952637, + "logps/chosen": -324.30859375, + "logps/rejected": -281.63848876953125, + "loss": 1810.1021, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13902036845684052, + "rewards/margins": 0.174498051404953, + "rewards/rejected": -0.3135184049606323, + "step": 7860 + }, + { + "epoch": 0.51, + "learning_rate": 2.805570668910041e-06, + "logits/chosen": -2.1317176818847656, + "logits/rejected": -2.0996460914611816, + "logps/chosen": -188.90228271484375, + "logps/rejected": -266.41033935546875, + "loss": 1824.0271, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.19262978434562683, + "rewards/margins": 0.14007696509361267, + "rewards/rejected": -0.3327067494392395, + "step": 7870 + }, + { + "epoch": 0.52, + "learning_rate": 2.7999027793415695e-06, + "logits/chosen": -2.5066134929656982, + "logits/rejected": -2.0417120456695557, + "logps/chosen": -257.93975830078125, + "logps/rejected": -228.1025848388672, + "loss": 1932.0182, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1302376091480255, + "rewards/margins": 0.1508847177028656, + "rewards/rejected": -0.2811223268508911, + "step": 7880 + }, + { + "epoch": 0.52, + "learning_rate": 2.794233325333293e-06, + "logits/chosen": -2.1897034645080566, + "logits/rejected": -2.0873708724975586, + "logps/chosen": -274.3140563964844, + "logps/rejected": -270.84112548828125, + "loss": 1531.2451, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16032613813877106, + "rewards/margins": 0.2020435333251953, + "rewards/rejected": -0.36236971616744995, + "step": 7890 + }, + { + "epoch": 0.52, + "learning_rate": 2.7885623364598597e-06, + "logits/chosen": -2.408759117126465, + "logits/rejected": -2.1068990230560303, + "logps/chosen": -278.29742431640625, + "logps/rejected": -260.3435974121094, + "loss": 1297.3996, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14604783058166504, + "rewards/margins": 0.23965105414390564, + "rewards/rejected": -0.3856988549232483, + "step": 7900 + }, + { + "epoch": 0.52, + "eval_logits/chosen": -2.313547134399414, + "eval_logits/rejected": -2.1292340755462646, + "eval_logps/chosen": -247.32740783691406, + "eval_logps/rejected": -243.37562561035156, + "eval_loss": 1798.07177734375, + "eval_rewards/accuracies": 0.6499999761581421, + "eval_rewards/chosen": -0.1532244086265564, + "eval_rewards/margins": 0.1644127070903778, + "eval_rewards/rejected": -0.3176371157169342, + "eval_runtime": 725.0357, + "eval_samples_per_second": 2.758, + "eval_steps_per_second": 1.379, + "step": 7900 + }, + { + "epoch": 0.52, + "learning_rate": 2.782889842303926e-06, + "logits/chosen": -2.2938106060028076, + "logits/rejected": -2.121316432952881, + "logps/chosen": -179.6522674560547, + "logps/rejected": -184.91526794433594, + "loss": 2068.6041, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.19423159956932068, + "rewards/margins": 0.10760875791311264, + "rewards/rejected": -0.3018403649330139, + "step": 7910 + }, + { + "epoch": 0.52, + "learning_rate": 2.7772158724559987e-06, + "logits/chosen": -2.118116855621338, + "logits/rejected": -1.9713754653930664, + "logps/chosen": -230.3015899658203, + "logps/rejected": -293.75885009765625, + "loss": 1291.2383, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1319253146648407, + "rewards/margins": 0.25986504554748535, + "rewards/rejected": -0.39179039001464844, + "step": 7920 + }, + { + "epoch": 0.52, + "learning_rate": 2.7715404565142856e-06, + "logits/chosen": -2.283179759979248, + "logits/rejected": -2.143989324569702, + "logps/chosen": -217.41445922851562, + "logps/rejected": -225.4498748779297, + "loss": 1855.3721, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1745026856660843, + "rewards/margins": 0.14258040487766266, + "rewards/rejected": -0.31708306074142456, + "step": 7930 + }, + { + "epoch": 0.52, + "learning_rate": 2.7658636240845354e-06, + "logits/chosen": -2.4289021492004395, + "logits/rejected": -2.3265390396118164, + "logps/chosen": -240.6270751953125, + "logps/rejected": -274.0114440917969, + "loss": 1486.8823, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17178744077682495, + "rewards/margins": 0.18766279518604279, + "rewards/rejected": -0.35945025086402893, + "step": 7940 + }, + { + "epoch": 0.52, + "learning_rate": 2.7601854047798872e-06, + "logits/chosen": -2.236783504486084, + "logits/rejected": -2.273075580596924, + "logps/chosen": -241.3152618408203, + "logps/rejected": -273.97210693359375, + "loss": 1882.7766, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.17391879856586456, + "rewards/margins": 0.13241559267044067, + "rewards/rejected": -0.3063344359397888, + "step": 7950 + }, + { + "epoch": 0.52, + "learning_rate": 2.7545058282207148e-06, + "logits/chosen": -2.384068489074707, + "logits/rejected": -1.9915987253189087, + "logps/chosen": -222.35446166992188, + "logps/rejected": -206.6442413330078, + "loss": 1766.518, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10812662541866302, + "rewards/margins": 0.14868710935115814, + "rewards/rejected": -0.2568137049674988, + "step": 7960 + }, + { + "epoch": 0.52, + "learning_rate": 2.748824924034471e-06, + "logits/chosen": -2.298893928527832, + "logits/rejected": -2.1646010875701904, + "logps/chosen": -241.891357421875, + "logps/rejected": -234.1492462158203, + "loss": 2235.9057, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.22481146454811096, + "rewards/margins": 0.11154550313949585, + "rewards/rejected": -0.3363569676876068, + "step": 7970 + }, + { + "epoch": 0.52, + "learning_rate": 2.743142721855536e-06, + "logits/chosen": -2.180006980895996, + "logits/rejected": -2.1479909420013428, + "logps/chosen": -166.78038024902344, + "logps/rejected": -169.9340362548828, + "loss": 1829.8742, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14920267462730408, + "rewards/margins": 0.11502110958099365, + "rewards/rejected": -0.26422375440597534, + "step": 7980 + }, + { + "epoch": 0.52, + "learning_rate": 2.737459251325058e-06, + "logits/chosen": -2.288517475128174, + "logits/rejected": -2.2369046211242676, + "logps/chosen": -282.2027587890625, + "logps/rejected": -271.6640625, + "loss": 1784.1377, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.12229214608669281, + "rewards/margins": 0.1199268251657486, + "rewards/rejected": -0.2422189712524414, + "step": 7990 + }, + { + "epoch": 0.52, + "learning_rate": 2.731774542090804e-06, + "logits/chosen": -2.2692532539367676, + "logits/rejected": -1.8465888500213623, + "logps/chosen": -209.0124969482422, + "logps/rejected": -196.2694091796875, + "loss": 2462.926, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.16236457228660583, + "rewards/margins": 0.06291927397251129, + "rewards/rejected": -0.22528386116027832, + "step": 8000 + }, + { + "epoch": 0.52, + "eval_logits/chosen": -2.326558828353882, + "eval_logits/rejected": -2.1409735679626465, + "eval_logps/chosen": -246.2211456298828, + "eval_logps/rejected": -240.75645446777344, + "eval_loss": 1784.8865966796875, + "eval_rewards/accuracies": 0.6554999947547913, + "eval_rewards/chosen": -0.14216183125972748, + "eval_rewards/margins": 0.1492837369441986, + "eval_rewards/rejected": -0.2914455533027649, + "eval_runtime": 726.2338, + "eval_samples_per_second": 2.754, + "eval_steps_per_second": 1.377, + "step": 8000 + }, + { + "epoch": 0.52, + "learning_rate": 2.7260886238070034e-06, + "logits/chosen": -2.3285775184631348, + "logits/rejected": -2.246692657470703, + "logps/chosen": -211.03939819335938, + "logps/rejected": -223.37353515625, + "loss": 1469.5437, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14569437503814697, + "rewards/margins": 0.17042239010334015, + "rewards/rejected": -0.3161167502403259, + "step": 8010 + }, + { + "epoch": 0.52, + "learning_rate": 2.72040152613419e-06, + "logits/chosen": -2.364499807357788, + "logits/rejected": -1.8563902378082275, + "logps/chosen": -232.80636596679688, + "logps/rejected": -167.91128540039062, + "loss": 1375.5312, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15931276977062225, + "rewards/margins": 0.18330441415309906, + "rewards/rejected": -0.3426171839237213, + "step": 8020 + }, + { + "epoch": 0.53, + "learning_rate": 2.7147132787390516e-06, + "logits/chosen": -2.3446335792541504, + "logits/rejected": -2.056989908218384, + "logps/chosen": -238.309326171875, + "logps/rejected": -240.9357452392578, + "loss": 1498.3412, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10831235349178314, + "rewards/margins": 0.19611985981464386, + "rewards/rejected": -0.304432213306427, + "step": 8030 + }, + { + "epoch": 0.53, + "learning_rate": 2.709023911294273e-06, + "logits/chosen": -2.429610013961792, + "logits/rejected": -1.9592183828353882, + "logps/chosen": -247.37881469726562, + "logps/rejected": -243.9899139404297, + "loss": 1500.2393, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.05059943348169327, + "rewards/margins": 0.27510666847229004, + "rewards/rejected": -0.3257060647010803, + "step": 8040 + }, + { + "epoch": 0.53, + "learning_rate": 2.7033334534783806e-06, + "logits/chosen": -2.321929454803467, + "logits/rejected": -2.4168248176574707, + "logps/chosen": -209.8223419189453, + "logps/rejected": -239.98876953125, + "loss": 2167.8779, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10676448047161102, + "rewards/margins": 0.1396731436252594, + "rewards/rejected": -0.24643762409687042, + "step": 8050 + }, + { + "epoch": 0.53, + "learning_rate": 2.697641934975592e-06, + "logits/chosen": -2.327284097671509, + "logits/rejected": -2.1021924018859863, + "logps/chosen": -236.8908233642578, + "logps/rejected": -221.6531982421875, + "loss": 1820.2213, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09104406088590622, + "rewards/margins": 0.17316171526908875, + "rewards/rejected": -0.26420578360557556, + "step": 8060 + }, + { + "epoch": 0.53, + "learning_rate": 2.691949385475654e-06, + "logits/chosen": -2.3733315467834473, + "logits/rejected": -2.116386890411377, + "logps/chosen": -255.9573974609375, + "logps/rejected": -245.44058227539062, + "loss": 1706.3115, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1206657662987709, + "rewards/margins": 0.15054434537887573, + "rewards/rejected": -0.2712101340293884, + "step": 8070 + }, + { + "epoch": 0.53, + "learning_rate": 2.6862558346736937e-06, + "logits/chosen": -2.2894411087036133, + "logits/rejected": -2.1502037048339844, + "logps/chosen": -250.6189727783203, + "logps/rejected": -265.82891845703125, + "loss": 1447.9819, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.12266738712787628, + "rewards/margins": 0.20411674678325653, + "rewards/rejected": -0.3267841637134552, + "step": 8080 + }, + { + "epoch": 0.53, + "learning_rate": 2.6805613122700617e-06, + "logits/chosen": -2.3372228145599365, + "logits/rejected": -1.9962005615234375, + "logps/chosen": -238.9376220703125, + "logps/rejected": -253.9717559814453, + "loss": 1820.5197, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1536514163017273, + "rewards/margins": 0.14091987907886505, + "rewards/rejected": -0.29457131028175354, + "step": 8090 + }, + { + "epoch": 0.53, + "learning_rate": 2.674865847970176e-06, + "logits/chosen": -2.2625794410705566, + "logits/rejected": -1.9928855895996094, + "logps/chosen": -219.21652221679688, + "logps/rejected": -254.426025390625, + "loss": 1812.3775, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.11600162833929062, + "rewards/margins": 0.12761160731315613, + "rewards/rejected": -0.24361324310302734, + "step": 8100 + }, + { + "epoch": 0.53, + "eval_logits/chosen": -2.320788621902466, + "eval_logits/rejected": -2.1356489658355713, + "eval_logps/chosen": -244.53488159179688, + "eval_logps/rejected": -238.00411987304688, + "eval_loss": 1789.9876708984375, + "eval_rewards/accuracies": 0.6545000076293945, + "eval_rewards/chosen": -0.12529927492141724, + "eval_rewards/margins": 0.13862305879592896, + "eval_rewards/rejected": -0.2639223337173462, + "eval_runtime": 724.9601, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.379, + "step": 8100 + }, + { + "epoch": 0.53, + "learning_rate": 2.669169471484368e-06, + "logits/chosen": -2.08323335647583, + "logits/rejected": -2.1388871669769287, + "logps/chosen": -182.0511474609375, + "logps/rejected": -188.3828887939453, + "loss": 2110.5072, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.16993075609207153, + "rewards/margins": 0.08971955627202988, + "rewards/rejected": -0.2596503496170044, + "step": 8110 + }, + { + "epoch": 0.53, + "learning_rate": 2.6634722125277278e-06, + "logits/chosen": -2.420374631881714, + "logits/rejected": -2.0989325046539307, + "logps/chosen": -248.5947723388672, + "logps/rejected": -268.76654052734375, + "loss": 1920.4986, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.16474846005439758, + "rewards/margins": 0.11976997554302216, + "rewards/rejected": -0.28451845049858093, + "step": 8120 + }, + { + "epoch": 0.53, + "learning_rate": 2.6577741008199498e-06, + "logits/chosen": -2.334933042526245, + "logits/rejected": -1.9777965545654297, + "logps/chosen": -268.9001770019531, + "logps/rejected": -229.83541870117188, + "loss": 1325.9265, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.12741991877555847, + "rewards/margins": 0.2097659558057785, + "rewards/rejected": -0.3371858596801758, + "step": 8130 + }, + { + "epoch": 0.53, + "learning_rate": 2.652075166085175e-06, + "logits/chosen": -2.222378969192505, + "logits/rejected": -2.175022602081299, + "logps/chosen": -246.1367645263672, + "logps/rejected": -289.9963684082031, + "loss": 1941.0016, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11557464301586151, + "rewards/margins": 0.2031438648700714, + "rewards/rejected": -0.31871849298477173, + "step": 8140 + }, + { + "epoch": 0.53, + "learning_rate": 2.6463754380518395e-06, + "logits/chosen": -2.2017464637756348, + "logits/rejected": -1.9937317371368408, + "logps/chosen": -247.4430694580078, + "logps/rejected": -205.15609741210938, + "loss": 1995.1008, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1648540049791336, + "rewards/margins": 0.12540122866630554, + "rewards/rejected": -0.29025521874427795, + "step": 8150 + }, + { + "epoch": 0.53, + "learning_rate": 2.6406749464525167e-06, + "logits/chosen": -2.306833028793335, + "logits/rejected": -2.032440423965454, + "logps/chosen": -237.3766632080078, + "logps/rejected": -207.55819702148438, + "loss": 1552.0191, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.03817574307322502, + "rewards/margins": 0.17413334548473358, + "rewards/rejected": -0.2123090922832489, + "step": 8160 + }, + { + "epoch": 0.53, + "learning_rate": 2.634973721023762e-06, + "logits/chosen": -2.347939968109131, + "logits/rejected": -2.1919798851013184, + "logps/chosen": -265.0765686035156, + "logps/rejected": -227.37551879882812, + "loss": 2049.8635, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1061432734131813, + "rewards/margins": 0.11737523972988129, + "rewards/rejected": -0.22351853549480438, + "step": 8170 + }, + { + "epoch": 0.54, + "learning_rate": 2.6292717915059605e-06, + "logits/chosen": -2.382143020629883, + "logits/rejected": -2.1191394329071045, + "logps/chosen": -285.326171875, + "logps/rejected": -247.8603515625, + "loss": 1394.937, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.09696097671985626, + "rewards/margins": 0.18727213144302368, + "rewards/rejected": -0.28423309326171875, + "step": 8180 + }, + { + "epoch": 0.54, + "learning_rate": 2.6235691876431706e-06, + "logits/chosen": -2.1982357501983643, + "logits/rejected": -2.253572702407837, + "logps/chosen": -231.0131378173828, + "logps/rejected": -249.4632110595703, + "loss": 1836.2434, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12397567927837372, + "rewards/margins": 0.12850920855998993, + "rewards/rejected": -0.25248488783836365, + "step": 8190 + }, + { + "epoch": 0.54, + "learning_rate": 2.6178659391829673e-06, + "logits/chosen": -2.4205422401428223, + "logits/rejected": -2.149384021759033, + "logps/chosen": -242.65371704101562, + "logps/rejected": -219.2117462158203, + "loss": 1606.6738, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.051757581532001495, + "rewards/margins": 0.17401257157325745, + "rewards/rejected": -0.22577011585235596, + "step": 8200 + }, + { + "epoch": 0.54, + "eval_logits/chosen": -2.302276372909546, + "eval_logits/rejected": -2.1190202236175537, + "eval_logps/chosen": -241.52880859375, + "eval_logps/rejected": -235.77371215820312, + "eval_loss": 1797.2037353515625, + "eval_rewards/accuracies": 0.6460000276565552, + "eval_rewards/chosen": -0.09523871541023254, + "eval_rewards/margins": 0.146379292011261, + "eval_rewards/rejected": -0.24161800742149353, + "eval_runtime": 722.3242, + "eval_samples_per_second": 2.769, + "eval_steps_per_second": 1.384, + "step": 8200 + }, + { + "epoch": 0.54, + "learning_rate": 2.6121620758762877e-06, + "logits/chosen": -2.289379596710205, + "logits/rejected": -2.0412538051605225, + "logps/chosen": -200.6814422607422, + "logps/rejected": -214.49612426757812, + "loss": 1785.8699, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.08597119152545929, + "rewards/margins": 0.14613738656044006, + "rewards/rejected": -0.23210859298706055, + "step": 8210 + }, + { + "epoch": 0.54, + "learning_rate": 2.606457627477277e-06, + "logits/chosen": -2.2264838218688965, + "logits/rejected": -2.172628402709961, + "logps/chosen": -181.49755859375, + "logps/rejected": -202.78858947753906, + "loss": 1664.8721, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.05868273228406906, + "rewards/margins": 0.17938938736915588, + "rewards/rejected": -0.23807211220264435, + "step": 8220 + }, + { + "epoch": 0.54, + "learning_rate": 2.6007526237431324e-06, + "logits/chosen": -2.3600382804870605, + "logits/rejected": -2.3124423027038574, + "logps/chosen": -189.7965850830078, + "logps/rejected": -219.19058227539062, + "loss": 1906.6195, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.07767064869403839, + "rewards/margins": 0.15275943279266357, + "rewards/rejected": -0.23043008148670197, + "step": 8230 + }, + { + "epoch": 0.54, + "learning_rate": 2.5950470944339478e-06, + "logits/chosen": -2.135626792907715, + "logits/rejected": -2.2016496658325195, + "logps/chosen": -228.05728149414062, + "logps/rejected": -234.7489776611328, + "loss": 1881.5312, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.06255332380533218, + "rewards/margins": 0.10187490284442902, + "rewards/rejected": -0.1644282042980194, + "step": 8240 + }, + { + "epoch": 0.54, + "learning_rate": 2.58934106931256e-06, + "logits/chosen": -2.2805557250976562, + "logits/rejected": -1.9872407913208008, + "logps/chosen": -231.88522338867188, + "logps/rejected": -226.14285278320312, + "loss": 2058.7807, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.11113202571868896, + "rewards/margins": 0.13435114920139313, + "rewards/rejected": -0.2454831600189209, + "step": 8250 + }, + { + "epoch": 0.54, + "learning_rate": 2.58363457814439e-06, + "logits/chosen": -2.2726893424987793, + "logits/rejected": -1.9903056621551514, + "logps/chosen": -227.5226593017578, + "logps/rejected": -227.06906127929688, + "loss": 2035.3699, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.16850681602954865, + "rewards/margins": 0.13269905745983124, + "rewards/rejected": -0.30120590329170227, + "step": 8260 + }, + { + "epoch": 0.54, + "learning_rate": 2.5779276506972924e-06, + "logits/chosen": -2.242635488510132, + "logits/rejected": -2.2251079082489014, + "logps/chosen": -245.4095458984375, + "logps/rejected": -219.79159545898438, + "loss": 1770.1719, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.134566992521286, + "rewards/margins": 0.1376422643661499, + "rewards/rejected": -0.2722092568874359, + "step": 8270 + }, + { + "epoch": 0.54, + "learning_rate": 2.5722203167413945e-06, + "logits/chosen": -2.3671107292175293, + "logits/rejected": -2.0390262603759766, + "logps/chosen": -294.7353515625, + "logps/rejected": -228.14974975585938, + "loss": 1792.4943, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11437664926052094, + "rewards/margins": 0.15335258841514587, + "rewards/rejected": -0.267729252576828, + "step": 8280 + }, + { + "epoch": 0.54, + "learning_rate": 2.5665126060489476e-06, + "logits/chosen": -2.3390555381774902, + "logits/rejected": -2.184868097305298, + "logps/chosen": -205.6299285888672, + "logps/rejected": -246.21206665039062, + "loss": 1913.0139, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.17527888715267181, + "rewards/margins": 0.11735512316226959, + "rewards/rejected": -0.2926340103149414, + "step": 8290 + }, + { + "epoch": 0.54, + "learning_rate": 2.560804548394165e-06, + "logits/chosen": -2.253669500350952, + "logits/rejected": -2.004037857055664, + "logps/chosen": -258.77728271484375, + "logps/rejected": -230.3441619873047, + "loss": 1769.8457, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11496938765048981, + "rewards/margins": 0.16839106380939484, + "rewards/rejected": -0.28336042165756226, + "step": 8300 + }, + { + "epoch": 0.54, + "eval_logits/chosen": -2.306878089904785, + "eval_logits/rejected": -2.1231863498687744, + "eval_logps/chosen": -242.7214813232422, + "eval_logps/rejected": -236.77609252929688, + "eval_loss": 1802.872802734375, + "eval_rewards/accuracies": 0.6449999809265137, + "eval_rewards/chosen": -0.10716545581817627, + "eval_rewards/margins": 0.1444765329360962, + "eval_rewards/rejected": -0.25164198875427246, + "eval_runtime": 726.3551, + "eval_samples_per_second": 2.753, + "eval_steps_per_second": 1.377, + "step": 8300 + }, + { + "epoch": 0.54, + "learning_rate": 2.5550961735530734e-06, + "logits/chosen": -2.142177104949951, + "logits/rejected": -2.322173595428467, + "logps/chosen": -167.76272583007812, + "logps/rejected": -215.3518524169922, + "loss": 1583.1814, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.047269057482481, + "rewards/margins": 0.16998426616191864, + "rewards/rejected": -0.21725329756736755, + "step": 8310 + }, + { + "epoch": 0.54, + "learning_rate": 2.549387511303351e-06, + "logits/chosen": -2.2949604988098145, + "logits/rejected": -2.3329718112945557, + "logps/chosen": -177.2271728515625, + "logps/rejected": -232.924072265625, + "loss": 1899.2404, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07490333169698715, + "rewards/margins": 0.11416280269622803, + "rewards/rejected": -0.18906612694263458, + "step": 8320 + }, + { + "epoch": 0.55, + "learning_rate": 2.5436785914241774e-06, + "logits/chosen": -2.250805377960205, + "logits/rejected": -2.2805335521698, + "logps/chosen": -211.9129638671875, + "logps/rejected": -195.1792449951172, + "loss": 1938.2619, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12953725457191467, + "rewards/margins": 0.15235944092273712, + "rewards/rejected": -0.281896710395813, + "step": 8330 + }, + { + "epoch": 0.55, + "learning_rate": 2.5379694436960746e-06, + "logits/chosen": -2.4175267219543457, + "logits/rejected": -2.2193617820739746, + "logps/chosen": -253.90762329101562, + "logps/rejected": -279.6899108886719, + "loss": 1799.4357, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0789659172296524, + "rewards/margins": 0.152174711227417, + "rewards/rejected": -0.2311406433582306, + "step": 8340 + }, + { + "epoch": 0.55, + "learning_rate": 2.5322600979007533e-06, + "logits/chosen": -2.4399263858795166, + "logits/rejected": -2.207584857940674, + "logps/chosen": -223.945556640625, + "logps/rejected": -214.1793975830078, + "loss": 1980.3602, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.11450109630823135, + "rewards/margins": 0.10933053493499756, + "rewards/rejected": -0.22383160889148712, + "step": 8350 + }, + { + "epoch": 0.55, + "learning_rate": 2.5265505838209592e-06, + "logits/chosen": -2.4518284797668457, + "logits/rejected": -2.104112148284912, + "logps/chosen": -272.073486328125, + "logps/rejected": -236.34579467773438, + "loss": 2270.6391, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15109841525554657, + "rewards/margins": 0.10685638338327408, + "rewards/rejected": -0.25795477628707886, + "step": 8360 + }, + { + "epoch": 0.55, + "learning_rate": 2.520840931240314e-06, + "logits/chosen": -2.4796156883239746, + "logits/rejected": -1.9970394372940063, + "logps/chosen": -222.74423217773438, + "logps/rejected": -175.6988067626953, + "loss": 1455.9042, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1293562650680542, + "rewards/margins": 0.1667545735836029, + "rewards/rejected": -0.2961108386516571, + "step": 8370 + }, + { + "epoch": 0.55, + "learning_rate": 2.515131169943162e-06, + "logits/chosen": -2.024977207183838, + "logits/rejected": -2.111084461212158, + "logps/chosen": -274.6491394042969, + "logps/rejected": -285.61962890625, + "loss": 1898.059, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16554591059684753, + "rewards/margins": 0.1794281303882599, + "rewards/rejected": -0.34497401118278503, + "step": 8380 + }, + { + "epoch": 0.55, + "learning_rate": 2.509421329714416e-06, + "logits/chosen": -2.1542716026306152, + "logits/rejected": -2.1845345497131348, + "logps/chosen": -221.7804718017578, + "logps/rejected": -252.6263885498047, + "loss": 1956.1996, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.15641272068023682, + "rewards/margins": 0.09653899818658829, + "rewards/rejected": -0.2529517412185669, + "step": 8390 + }, + { + "epoch": 0.55, + "learning_rate": 2.5037114403393987e-06, + "logits/chosen": -2.2747371196746826, + "logits/rejected": -2.02976655960083, + "logps/chosen": -220.5301971435547, + "logps/rejected": -201.79635620117188, + "loss": 1652.7957, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09549812972545624, + "rewards/margins": 0.1504668891429901, + "rewards/rejected": -0.24596503376960754, + "step": 8400 + }, + { + "epoch": 0.55, + "eval_logits/chosen": -2.3137805461883545, + "eval_logits/rejected": -2.1298303604125977, + "eval_logps/chosen": -247.04515075683594, + "eval_logps/rejected": -241.276611328125, + "eval_loss": 1804.0986328125, + "eval_rewards/accuracies": 0.6460000276565552, + "eval_rewards/chosen": -0.1504022628068924, + "eval_rewards/margins": 0.14624497294425964, + "eval_rewards/rejected": -0.29664725065231323, + "eval_runtime": 722.8032, + "eval_samples_per_second": 2.767, + "eval_steps_per_second": 1.384, + "step": 8400 + }, + { + "epoch": 0.55, + "learning_rate": 2.4980015316036908e-06, + "logits/chosen": -2.160120964050293, + "logits/rejected": -2.2130343914031982, + "logps/chosen": -187.4803466796875, + "logps/rejected": -232.2801055908203, + "loss": 1326.5803, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11670643091201782, + "rewards/margins": 0.23213955760002136, + "rewards/rejected": -0.34884604811668396, + "step": 8410 + }, + { + "epoch": 0.55, + "learning_rate": 2.4922916332929725e-06, + "logits/chosen": -2.4907407760620117, + "logits/rejected": -2.2238717079162598, + "logps/chosen": -248.92971801757812, + "logps/rejected": -221.5033721923828, + "loss": 2002.0156, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.13505470752716064, + "rewards/margins": 0.12188147008419037, + "rewards/rejected": -0.2569361925125122, + "step": 8420 + }, + { + "epoch": 0.55, + "learning_rate": 2.4865817751928716e-06, + "logits/chosen": -2.228563070297241, + "logits/rejected": -2.216607093811035, + "logps/chosen": -210.0905303955078, + "logps/rejected": -256.52838134765625, + "loss": 1457.9258, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14601817727088928, + "rewards/margins": 0.20015673339366913, + "rewards/rejected": -0.3461748957633972, + "step": 8430 + }, + { + "epoch": 0.55, + "learning_rate": 2.4808719870888037e-06, + "logits/chosen": -2.0969958305358887, + "logits/rejected": -2.0262608528137207, + "logps/chosen": -233.91207885742188, + "logps/rejected": -217.42391967773438, + "loss": 1678.0998, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16240623593330383, + "rewards/margins": 0.167855903506279, + "rewards/rejected": -0.33026212453842163, + "step": 8440 + }, + { + "epoch": 0.55, + "learning_rate": 2.4751622987658206e-06, + "logits/chosen": -2.5014309883117676, + "logits/rejected": -2.3213982582092285, + "logps/chosen": -249.9745330810547, + "logps/rejected": -253.134765625, + "loss": 1850.9133, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12024291604757309, + "rewards/margins": 0.13900864124298096, + "rewards/rejected": -0.25925153493881226, + "step": 8450 + }, + { + "epoch": 0.55, + "learning_rate": 2.4694527400084546e-06, + "logits/chosen": -2.286902904510498, + "logits/rejected": -2.1964993476867676, + "logps/chosen": -235.66690063476562, + "logps/rejected": -243.6966552734375, + "loss": 1702.9672, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13422232866287231, + "rewards/margins": 0.1387438029050827, + "rewards/rejected": -0.2729661464691162, + "step": 8460 + }, + { + "epoch": 0.55, + "learning_rate": 2.4637433406005607e-06, + "logits/chosen": -2.4860024452209473, + "logits/rejected": -2.370863437652588, + "logps/chosen": -323.28936767578125, + "logps/rejected": -309.1042175292969, + "loss": 1832.9215, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13563047349452972, + "rewards/margins": 0.13858014345169067, + "rewards/rejected": -0.2742106318473816, + "step": 8470 + }, + { + "epoch": 0.55, + "learning_rate": 2.4580341303251628e-06, + "logits/chosen": -2.301816940307617, + "logits/rejected": -2.037771701812744, + "logps/chosen": -272.97430419921875, + "logps/rejected": -257.2006530761719, + "loss": 1560.2741, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13023129105567932, + "rewards/margins": 0.20625126361846924, + "rewards/rejected": -0.33648252487182617, + "step": 8480 + }, + { + "epoch": 0.56, + "learning_rate": 2.4523251389642984e-06, + "logits/chosen": -2.1981709003448486, + "logits/rejected": -2.0751469135284424, + "logps/chosen": -271.48138427734375, + "logps/rejected": -254.32119750976562, + "loss": 1673.4539, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16922123730182648, + "rewards/margins": 0.19978253543376923, + "rewards/rejected": -0.3690037429332733, + "step": 8490 + }, + { + "epoch": 0.56, + "learning_rate": 2.4466163962988626e-06, + "logits/chosen": -2.5145349502563477, + "logits/rejected": -2.145453691482544, + "logps/chosen": -297.2919006347656, + "logps/rejected": -214.80752563476562, + "loss": 2091.1088, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1669091135263443, + "rewards/margins": 0.1623493880033493, + "rewards/rejected": -0.3292585015296936, + "step": 8500 + }, + { + "epoch": 0.56, + "eval_logits/chosen": -2.3224895000457764, + "eval_logits/rejected": -2.1371631622314453, + "eval_logps/chosen": -247.69615173339844, + "eval_logps/rejected": -243.36134338378906, + "eval_loss": 1803.540771484375, + "eval_rewards/accuracies": 0.6510000228881836, + "eval_rewards/chosen": -0.15691210329532623, + "eval_rewards/margins": 0.16058222949504852, + "eval_rewards/rejected": -0.31749436259269714, + "eval_runtime": 723.5499, + "eval_samples_per_second": 2.764, + "eval_steps_per_second": 1.382, + "step": 8500 + }, + { + "epoch": 0.56, + "learning_rate": 2.4409079321084543e-06, + "logits/chosen": -2.273221492767334, + "logits/rejected": -2.324763298034668, + "logps/chosen": -227.0233154296875, + "logps/rejected": -272.7955017089844, + "loss": 1877.4422, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11903639882802963, + "rewards/margins": 0.1583327353000641, + "rewards/rejected": -0.2773691713809967, + "step": 8510 + }, + { + "epoch": 0.56, + "learning_rate": 2.4351997761712184e-06, + "logits/chosen": -2.522839069366455, + "logits/rejected": -2.0727713108062744, + "logps/chosen": -257.5574645996094, + "logps/rejected": -212.2241973876953, + "loss": 1262.9665, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12169145047664642, + "rewards/margins": 0.2137262374162674, + "rewards/rejected": -0.33541765809059143, + "step": 8520 + }, + { + "epoch": 0.56, + "learning_rate": 2.4294919582636933e-06, + "logits/chosen": -2.3109967708587646, + "logits/rejected": -2.169250965118408, + "logps/chosen": -223.28262329101562, + "logps/rejected": -228.7783966064453, + "loss": 1521.7472, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12180079519748688, + "rewards/margins": 0.17335185408592224, + "rewards/rejected": -0.29515260457992554, + "step": 8530 + }, + { + "epoch": 0.56, + "learning_rate": 2.423784508160652e-06, + "logits/chosen": -2.3863461017608643, + "logits/rejected": -2.138375759124756, + "logps/chosen": -272.7332458496094, + "logps/rejected": -240.0067138671875, + "loss": 1791.726, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.17502503097057343, + "rewards/margins": 0.15110823512077332, + "rewards/rejected": -0.32613325119018555, + "step": 8540 + }, + { + "epoch": 0.56, + "learning_rate": 2.418077455634951e-06, + "logits/chosen": -2.21634578704834, + "logits/rejected": -2.2655911445617676, + "logps/chosen": -227.6550750732422, + "logps/rejected": -268.90667724609375, + "loss": 1871.1775, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11083988845348358, + "rewards/margins": 0.13694430887699127, + "rewards/rejected": -0.24778418242931366, + "step": 8550 + }, + { + "epoch": 0.56, + "learning_rate": 2.4123708304573714e-06, + "logits/chosen": -2.3985753059387207, + "logits/rejected": -2.2608468532562256, + "logps/chosen": -299.736083984375, + "logps/rejected": -301.4363098144531, + "loss": 1622.1048, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09968741238117218, + "rewards/margins": 0.18398933112621307, + "rewards/rejected": -0.28367674350738525, + "step": 8560 + }, + { + "epoch": 0.56, + "learning_rate": 2.406664662396465e-06, + "logits/chosen": -2.191650152206421, + "logits/rejected": -2.0365116596221924, + "logps/chosen": -202.6273651123047, + "logps/rejected": -198.66024780273438, + "loss": 2089.2441, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.18245482444763184, + "rewards/margins": 0.11715099960565567, + "rewards/rejected": -0.2996058464050293, + "step": 8570 + }, + { + "epoch": 0.56, + "learning_rate": 2.4009589812184012e-06, + "logits/chosen": -2.357086181640625, + "logits/rejected": -1.9754444360733032, + "logps/chosen": -217.845947265625, + "logps/rejected": -177.6644287109375, + "loss": 1696.61, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14626047015190125, + "rewards/margins": 0.12929466366767883, + "rewards/rejected": -0.2755551338195801, + "step": 8580 + }, + { + "epoch": 0.56, + "learning_rate": 2.3952538166868073e-06, + "logits/chosen": -2.1383872032165527, + "logits/rejected": -2.198594093322754, + "logps/chosen": -230.4532928466797, + "logps/rejected": -234.652587890625, + "loss": 1386.3909, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1290692389011383, + "rewards/margins": 0.2079249918460846, + "rewards/rejected": -0.3369942307472229, + "step": 8590 + }, + { + "epoch": 0.56, + "learning_rate": 2.389549198562616e-06, + "logits/chosen": -2.3295209407806396, + "logits/rejected": -1.9044430255889893, + "logps/chosen": -239.19851684570312, + "logps/rejected": -229.85903930664062, + "loss": 1515.3847, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13116881251335144, + "rewards/margins": 0.20640698075294495, + "rewards/rejected": -0.337575763463974, + "step": 8600 + }, + { + "epoch": 0.56, + "eval_logits/chosen": -2.3367018699645996, + "eval_logits/rejected": -2.1502957344055176, + "eval_logps/chosen": -244.80181884765625, + "eval_logps/rejected": -240.16551208496094, + "eval_loss": 1784.409423828125, + "eval_rewards/accuracies": 0.656000018119812, + "eval_rewards/chosen": -0.12796856462955475, + "eval_rewards/margins": 0.15756747126579285, + "eval_rewards/rejected": -0.2855360507965088, + "eval_runtime": 724.6062, + "eval_samples_per_second": 2.76, + "eval_steps_per_second": 1.38, + "step": 8600 + }, + { + "epoch": 0.56, + "learning_rate": 2.3838451566039098e-06, + "logits/chosen": -2.3467819690704346, + "logits/rejected": -2.1745972633361816, + "logps/chosen": -256.1773376464844, + "logps/rejected": -255.5081329345703, + "loss": 2093.549, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17667360603809357, + "rewards/margins": 0.09464854001998901, + "rewards/rejected": -0.2713221311569214, + "step": 8610 + }, + { + "epoch": 0.56, + "learning_rate": 2.3781417205657662e-06, + "logits/chosen": -2.36430025100708, + "logits/rejected": -2.0601418018341064, + "logps/chosen": -210.05319213867188, + "logps/rejected": -189.75921630859375, + "loss": 1675.3691, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12296760082244873, + "rewards/margins": 0.17913182079792023, + "rewards/rejected": -0.30209943652153015, + "step": 8620 + }, + { + "epoch": 0.56, + "learning_rate": 2.3724389202001006e-06, + "logits/chosen": -2.393179178237915, + "logits/rejected": -2.12575101852417, + "logps/chosen": -219.45242309570312, + "logps/rejected": -213.09487915039062, + "loss": 1820.0014, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.17128977179527283, + "rewards/margins": 0.15339143574237823, + "rewards/rejected": -0.32468119263648987, + "step": 8630 + }, + { + "epoch": 0.57, + "learning_rate": 2.366736785255514e-06, + "logits/chosen": -2.276599168777466, + "logits/rejected": -2.2039132118225098, + "logps/chosen": -214.40023803710938, + "logps/rejected": -216.3034210205078, + "loss": 1842.123, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.15671400725841522, + "rewards/margins": 0.13123196363449097, + "rewards/rejected": -0.2879459857940674, + "step": 8640 + }, + { + "epoch": 0.57, + "learning_rate": 2.3610353454771355e-06, + "logits/chosen": -2.183971881866455, + "logits/rejected": -2.1115612983703613, + "logps/chosen": -205.38037109375, + "logps/rejected": -202.2388916015625, + "loss": 2108.0105, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.15432950854301453, + "rewards/margins": 0.13153082132339478, + "rewards/rejected": -0.2858603298664093, + "step": 8650 + }, + { + "epoch": 0.57, + "learning_rate": 2.355334630606467e-06, + "logits/chosen": -2.533132791519165, + "logits/rejected": -2.052190065383911, + "logps/chosen": -257.5996398925781, + "logps/rejected": -207.3619842529297, + "loss": 1800.7424, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1730933040380478, + "rewards/margins": 0.13307951390743256, + "rewards/rejected": -0.30617281794548035, + "step": 8660 + }, + { + "epoch": 0.57, + "learning_rate": 2.349634670381231e-06, + "logits/chosen": -2.1400325298309326, + "logits/rejected": -2.0968689918518066, + "logps/chosen": -222.1932830810547, + "logps/rejected": -245.26327514648438, + "loss": 1916.9623, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14994195103645325, + "rewards/margins": 0.1327134072780609, + "rewards/rejected": -0.2826553285121918, + "step": 8670 + }, + { + "epoch": 0.57, + "learning_rate": 2.3439354945352104e-06, + "logits/chosen": -2.387784719467163, + "logits/rejected": -2.3289902210235596, + "logps/chosen": -262.81951904296875, + "logps/rejected": -225.038818359375, + "loss": 2254.6959, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.19295933842658997, + "rewards/margins": 0.06617043167352676, + "rewards/rejected": -0.2591297924518585, + "step": 8680 + }, + { + "epoch": 0.57, + "learning_rate": 2.3382371327981e-06, + "logits/chosen": -2.2475638389587402, + "logits/rejected": -2.2487053871154785, + "logps/chosen": -244.71591186523438, + "logps/rejected": -249.59848022460938, + "loss": 1358.0566, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1323966681957245, + "rewards/margins": 0.1917734146118164, + "rewards/rejected": -0.3241700828075409, + "step": 8690 + }, + { + "epoch": 0.57, + "learning_rate": 2.3325396148953456e-06, + "logits/chosen": -2.1244850158691406, + "logits/rejected": -2.2361392974853516, + "logps/chosen": -189.0052490234375, + "logps/rejected": -257.8404846191406, + "loss": 1773.4947, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1770378202199936, + "rewards/margins": 0.14704987406730652, + "rewards/rejected": -0.3240876793861389, + "step": 8700 + }, + { + "epoch": 0.57, + "eval_logits/chosen": -2.3439927101135254, + "eval_logits/rejected": -2.1572177410125732, + "eval_logps/chosen": -246.85609436035156, + "eval_logps/rejected": -241.23651123046875, + "eval_loss": 1781.3251953125, + "eval_rewards/accuracies": 0.6539999842643738, + "eval_rewards/chosen": -0.14851143956184387, + "eval_rewards/margins": 0.14773471653461456, + "eval_rewards/rejected": -0.29624614119529724, + "eval_runtime": 724.3642, + "eval_samples_per_second": 2.761, + "eval_steps_per_second": 1.381, + "step": 8700 + }, + { + "epoch": 0.57, + "learning_rate": 2.3268429705479915e-06, + "logits/chosen": -2.5082974433898926, + "logits/rejected": -2.1605987548828125, + "logps/chosen": -236.01535034179688, + "logps/rejected": -214.6260528564453, + "loss": 1315.2974, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11202754825353622, + "rewards/margins": 0.19929727911949158, + "rewards/rejected": -0.3113248348236084, + "step": 8710 + }, + { + "epoch": 0.57, + "learning_rate": 2.3211472294725248e-06, + "logits/chosen": -2.3658642768859863, + "logits/rejected": -2.23079252243042, + "logps/chosen": -230.264892578125, + "logps/rejected": -236.5784149169922, + "loss": 1565.0962, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13849703967571259, + "rewards/margins": 0.18980672955513, + "rewards/rejected": -0.328303724527359, + "step": 8720 + }, + { + "epoch": 0.57, + "learning_rate": 2.315452421380721e-06, + "logits/chosen": -2.249817132949829, + "logits/rejected": -1.7914634943008423, + "logps/chosen": -268.02972412109375, + "logps/rejected": -234.94607543945312, + "loss": 1925.0975, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.13085412979125977, + "rewards/margins": 0.14909395575523376, + "rewards/rejected": -0.27994805574417114, + "step": 8730 + }, + { + "epoch": 0.57, + "learning_rate": 2.3097585759794886e-06, + "logits/chosen": -2.3426501750946045, + "logits/rejected": -1.9522781372070312, + "logps/chosen": -266.3086853027344, + "logps/rejected": -224.4796905517578, + "loss": 1628.2115, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.11967410892248154, + "rewards/margins": 0.21562986075878143, + "rewards/rejected": -0.3353039622306824, + "step": 8740 + }, + { + "epoch": 0.57, + "learning_rate": 2.3040657229707155e-06, + "logits/chosen": -2.352038621902466, + "logits/rejected": -2.247140884399414, + "logps/chosen": -187.58377075195312, + "logps/rejected": -216.38687133789062, + "loss": 1605.9856, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15536166727542877, + "rewards/margins": 0.17247450351715088, + "rewards/rejected": -0.32783618569374084, + "step": 8750 + }, + { + "epoch": 0.57, + "learning_rate": 2.2983738920511104e-06, + "logits/chosen": -2.513679027557373, + "logits/rejected": -2.0306098461151123, + "logps/chosen": -281.51922607421875, + "logps/rejected": -248.924072265625, + "loss": 1550.2564, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1172635406255722, + "rewards/margins": 0.17706869542598724, + "rewards/rejected": -0.29433223605155945, + "step": 8760 + }, + { + "epoch": 0.57, + "learning_rate": 2.2926831129120523e-06, + "logits/chosen": -2.165912389755249, + "logits/rejected": -2.1230220794677734, + "logps/chosen": -249.427734375, + "logps/rejected": -236.6632537841797, + "loss": 1452.7133, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.13636116683483124, + "rewards/margins": 0.1559121161699295, + "rewards/rejected": -0.29227328300476074, + "step": 8770 + }, + { + "epoch": 0.57, + "learning_rate": 2.2869934152394323e-06, + "logits/chosen": -2.3656163215637207, + "logits/rejected": -2.095022201538086, + "logps/chosen": -284.83721923828125, + "logps/rejected": -240.811767578125, + "loss": 2238.9471, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.15871794521808624, + "rewards/margins": 0.11576612293720245, + "rewards/rejected": -0.2744840681552887, + "step": 8780 + }, + { + "epoch": 0.58, + "learning_rate": 2.281304828713501e-06, + "logits/chosen": -2.2470030784606934, + "logits/rejected": -2.1870293617248535, + "logps/chosen": -251.87753295898438, + "logps/rejected": -257.03125, + "loss": 1946.543, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.18890242278575897, + "rewards/margins": 0.12656234204769135, + "rewards/rejected": -0.3154647648334503, + "step": 8790 + }, + { + "epoch": 0.58, + "learning_rate": 2.275617383008711e-06, + "logits/chosen": -2.341176986694336, + "logits/rejected": -2.240370512008667, + "logps/chosen": -250.6907501220703, + "logps/rejected": -261.1102600097656, + "loss": 1795.5312, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.15203282237052917, + "rewards/margins": 0.13087281584739685, + "rewards/rejected": -0.282905638217926, + "step": 8800 + }, + { + "epoch": 0.58, + "eval_logits/chosen": -2.3469302654266357, + "eval_logits/rejected": -2.1595752239227295, + "eval_logps/chosen": -243.88812255859375, + "eval_logps/rejected": -239.34646606445312, + "eval_loss": 1785.9962158203125, + "eval_rewards/accuracies": 0.6585000157356262, + "eval_rewards/chosen": -0.11883172392845154, + "eval_rewards/margins": 0.1585141122341156, + "eval_rewards/rejected": -0.27734580636024475, + "eval_runtime": 724.8827, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 8800 + }, + { + "epoch": 0.58, + "learning_rate": 2.269931107793567e-06, + "logits/chosen": -2.271245002746582, + "logits/rejected": -2.180269718170166, + "logps/chosen": -222.31216430664062, + "logps/rejected": -243.32568359375, + "loss": 1879.1926, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1003180518746376, + "rewards/margins": 0.1292106956243515, + "rewards/rejected": -0.2295287549495697, + "step": 8810 + }, + { + "epoch": 0.58, + "learning_rate": 2.2642460327304655e-06, + "logits/chosen": -2.1947274208068848, + "logits/rejected": -2.2527339458465576, + "logps/chosen": -255.667236328125, + "logps/rejected": -253.3027801513672, + "loss": 2138.826, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.11107480525970459, + "rewards/margins": 0.1348305344581604, + "rewards/rejected": -0.2459053248167038, + "step": 8820 + }, + { + "epoch": 0.58, + "learning_rate": 2.258562187475543e-06, + "logits/chosen": -2.1851754188537598, + "logits/rejected": -2.1546683311462402, + "logps/chosen": -238.5567626953125, + "logps/rejected": -212.98873901367188, + "loss": 1958.4557, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11609867960214615, + "rewards/margins": 0.11943832784891129, + "rewards/rejected": -0.23553700745105743, + "step": 8830 + }, + { + "epoch": 0.58, + "learning_rate": 2.2528796016785196e-06, + "logits/chosen": -2.246023654937744, + "logits/rejected": -2.0586771965026855, + "logps/chosen": -204.19004821777344, + "logps/rejected": -231.16177368164062, + "loss": 1600.6956, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14272116124629974, + "rewards/margins": 0.17305660247802734, + "rewards/rejected": -0.3157777786254883, + "step": 8840 + }, + { + "epoch": 0.58, + "learning_rate": 2.247198304982548e-06, + "logits/chosen": -2.2854185104370117, + "logits/rejected": -2.0803418159484863, + "logps/chosen": -174.22653198242188, + "logps/rejected": -184.52342224121094, + "loss": 1828.7586, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1225501298904419, + "rewards/margins": 0.1424633413553238, + "rewards/rejected": -0.2650134861469269, + "step": 8850 + }, + { + "epoch": 0.58, + "learning_rate": 2.2415183270240533e-06, + "logits/chosen": -2.54646635055542, + "logits/rejected": -2.302757501602173, + "logps/chosen": -212.83834838867188, + "logps/rejected": -233.5038604736328, + "loss": 1681.2141, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1350872814655304, + "rewards/margins": 0.16359901428222656, + "rewards/rejected": -0.29868629574775696, + "step": 8860 + }, + { + "epoch": 0.58, + "learning_rate": 2.2358396974325837e-06, + "logits/chosen": -2.35323429107666, + "logits/rejected": -2.1477296352386475, + "logps/chosen": -250.175048828125, + "logps/rejected": -240.62460327148438, + "loss": 1567.8297, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.07907464355230331, + "rewards/margins": 0.16716596484184265, + "rewards/rejected": -0.24624061584472656, + "step": 8870 + }, + { + "epoch": 0.58, + "learning_rate": 2.2301624458306525e-06, + "logits/chosen": -2.449899196624756, + "logits/rejected": -2.160046339035034, + "logps/chosen": -275.97906494140625, + "logps/rejected": -235.339599609375, + "loss": 2116.3758, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16630972921848297, + "rewards/margins": 0.10701718181371689, + "rewards/rejected": -0.27332693338394165, + "step": 8880 + }, + { + "epoch": 0.58, + "learning_rate": 2.2244866018335855e-06, + "logits/chosen": -2.305452823638916, + "logits/rejected": -2.298046350479126, + "logps/chosen": -229.87179565429688, + "logps/rejected": -253.76742553710938, + "loss": 2342.0385, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.12814125418663025, + "rewards/margins": 0.09488587826490402, + "rewards/rejected": -0.22302713990211487, + "step": 8890 + }, + { + "epoch": 0.58, + "learning_rate": 2.2188121950493648e-06, + "logits/chosen": -2.44734263420105, + "logits/rejected": -2.075795888900757, + "logps/chosen": -233.55419921875, + "logps/rejected": -167.12826538085938, + "loss": 1880.782, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1481078565120697, + "rewards/margins": 0.11826397478580475, + "rewards/rejected": -0.26637178659439087, + "step": 8900 + }, + { + "epoch": 0.58, + "eval_logits/chosen": -2.344513177871704, + "eval_logits/rejected": -2.157212972640991, + "eval_logps/chosen": -246.5005340576172, + "eval_logps/rejected": -242.08053588867188, + "eval_loss": 1782.6387939453125, + "eval_rewards/accuracies": 0.6545000076293945, + "eval_rewards/chosen": -0.14495587348937988, + "eval_rewards/margins": 0.15973049402236938, + "eval_rewards/rejected": -0.30468639731407166, + "eval_runtime": 723.8534, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.381, + "step": 8900 + }, + { + "epoch": 0.58, + "learning_rate": 2.2131392550784766e-06, + "logits/chosen": -2.464691638946533, + "logits/rejected": -1.890479326248169, + "logps/chosen": -298.20574951171875, + "logps/rejected": -222.00830078125, + "loss": 1379.1506, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13327114284038544, + "rewards/margins": 0.19366781413555145, + "rewards/rejected": -0.3269389569759369, + "step": 8910 + }, + { + "epoch": 0.58, + "learning_rate": 2.2074678115137533e-06, + "logits/chosen": -2.1455368995666504, + "logits/rejected": -2.0546436309814453, + "logps/chosen": -208.8870391845703, + "logps/rejected": -233.684814453125, + "loss": 1581.221, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1417008638381958, + "rewards/margins": 0.19881223142147064, + "rewards/rejected": -0.34051311016082764, + "step": 8920 + }, + { + "epoch": 0.58, + "learning_rate": 2.201797893940224e-06, + "logits/chosen": -2.2259609699249268, + "logits/rejected": -2.015672445297241, + "logps/chosen": -245.16067504882812, + "logps/rejected": -282.9098815917969, + "loss": 1605.6056, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11983288824558258, + "rewards/margins": 0.1739863008260727, + "rewards/rejected": -0.2938191592693329, + "step": 8930 + }, + { + "epoch": 0.58, + "learning_rate": 2.196129531934956e-06, + "logits/chosen": -2.27424955368042, + "logits/rejected": -2.0044965744018555, + "logps/chosen": -248.17770385742188, + "logps/rejected": -248.91513061523438, + "loss": 1630.4396, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10263688862323761, + "rewards/margins": 0.1879998743534088, + "rewards/rejected": -0.29063680768013, + "step": 8940 + }, + { + "epoch": 0.59, + "learning_rate": 2.190462755066902e-06, + "logits/chosen": -2.280273199081421, + "logits/rejected": -2.0560171604156494, + "logps/chosen": -280.7376403808594, + "logps/rejected": -269.51123046875, + "loss": 1606.276, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.183364599943161, + "rewards/margins": 0.1426433026790619, + "rewards/rejected": -0.3260079026222229, + "step": 8950 + }, + { + "epoch": 0.59, + "learning_rate": 2.184797592896746e-06, + "logits/chosen": -2.4171805381774902, + "logits/rejected": -2.382488250732422, + "logps/chosen": -245.8582763671875, + "logps/rejected": -236.25991821289062, + "loss": 1587.3939, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11422693729400635, + "rewards/margins": 0.16934755444526672, + "rewards/rejected": -0.28357452154159546, + "step": 8960 + }, + { + "epoch": 0.59, + "learning_rate": 2.17913407497675e-06, + "logits/chosen": -2.3496878147125244, + "logits/rejected": -2.4273006916046143, + "logps/chosen": -185.42147827148438, + "logps/rejected": -237.0035400390625, + "loss": 1905.2375, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.0703950971364975, + "rewards/margins": 0.1763085126876831, + "rewards/rejected": -0.24670365452766418, + "step": 8970 + }, + { + "epoch": 0.59, + "learning_rate": 2.173472230850596e-06, + "logits/chosen": -2.462353229522705, + "logits/rejected": -2.274624824523926, + "logps/chosen": -212.4112091064453, + "logps/rejected": -182.86685180664062, + "loss": 2017.8213, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.12148640304803848, + "rewards/margins": 0.10899336636066437, + "rewards/rejected": -0.23047976195812225, + "step": 8980 + }, + { + "epoch": 0.59, + "learning_rate": 2.1678120900532375e-06, + "logits/chosen": -2.44746732711792, + "logits/rejected": -2.1482653617858887, + "logps/chosen": -251.8532257080078, + "logps/rejected": -239.1935577392578, + "loss": 1699.1285, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1593838632106781, + "rewards/margins": 0.16600628197193146, + "rewards/rejected": -0.32539016008377075, + "step": 8990 + }, + { + "epoch": 0.59, + "learning_rate": 2.1621536821107412e-06, + "logits/chosen": -2.332921266555786, + "logits/rejected": -2.205714702606201, + "logps/chosen": -214.71414184570312, + "logps/rejected": -189.07632446289062, + "loss": 1695.9539, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12144309282302856, + "rewards/margins": 0.1639450341463089, + "rewards/rejected": -0.28538811206817627, + "step": 9000 + }, + { + "epoch": 0.59, + "eval_logits/chosen": -2.348705768585205, + "eval_logits/rejected": -2.161200761795044, + "eval_logps/chosen": -243.29458618164062, + "eval_logps/rejected": -239.52146911621094, + "eval_loss": 1783.7203369140625, + "eval_rewards/accuracies": 0.6625000238418579, + "eval_rewards/chosen": -0.11289641261100769, + "eval_rewards/margins": 0.16619925200939178, + "eval_rewards/rejected": -0.27909567952156067, + "eval_runtime": 723.4332, + "eval_samples_per_second": 2.765, + "eval_steps_per_second": 1.382, + "step": 9000 + }, + { + "epoch": 0.59, + "learning_rate": 2.1564970365401346e-06, + "logits/chosen": -2.3792998790740967, + "logits/rejected": -2.1225550174713135, + "logps/chosen": -200.08950805664062, + "logps/rejected": -175.19876098632812, + "loss": 2113.7773, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.15938423573970795, + "rewards/margins": 0.13117334246635437, + "rewards/rejected": -0.2905575633049011, + "step": 9010 + }, + { + "epoch": 0.59, + "learning_rate": 2.1508421828492527e-06, + "logits/chosen": -2.501051425933838, + "logits/rejected": -2.158234119415283, + "logps/chosen": -235.07876586914062, + "logps/rejected": -181.7694854736328, + "loss": 1914.55, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.09886038303375244, + "rewards/margins": 0.14931032061576843, + "rewards/rejected": -0.24817068874835968, + "step": 9020 + }, + { + "epoch": 0.59, + "learning_rate": 2.145189150536582e-06, + "logits/chosen": -2.177340030670166, + "logits/rejected": -2.081105947494507, + "logps/chosen": -232.1053924560547, + "logps/rejected": -197.1642303466797, + "loss": 1794.2258, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0993586927652359, + "rewards/margins": 0.14461085200309753, + "rewards/rejected": -0.24396955966949463, + "step": 9030 + }, + { + "epoch": 0.59, + "learning_rate": 2.139537969091107e-06, + "logits/chosen": -2.2126095294952393, + "logits/rejected": -2.1825172901153564, + "logps/chosen": -277.8831481933594, + "logps/rejected": -227.9888458251953, + "loss": 2110.9107, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1229945570230484, + "rewards/margins": 0.12276256084442139, + "rewards/rejected": -0.2457571029663086, + "step": 9040 + }, + { + "epoch": 0.59, + "learning_rate": 2.1338886679921603e-06, + "logits/chosen": -2.280548572540283, + "logits/rejected": -2.215135097503662, + "logps/chosen": -252.32119750976562, + "logps/rejected": -245.0319061279297, + "loss": 1867.8156, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1077810525894165, + "rewards/margins": 0.13121111690998077, + "rewards/rejected": -0.23899218440055847, + "step": 9050 + }, + { + "epoch": 0.59, + "learning_rate": 2.128241276709263e-06, + "logits/chosen": -2.361011028289795, + "logits/rejected": -2.3018691539764404, + "logps/chosen": -209.04067993164062, + "logps/rejected": -245.54940795898438, + "loss": 1783.9053, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.05778917670249939, + "rewards/margins": 0.1647137701511383, + "rewards/rejected": -0.2225029468536377, + "step": 9060 + }, + { + "epoch": 0.59, + "learning_rate": 2.1225958247019746e-06, + "logits/chosen": -2.389866590499878, + "logits/rejected": -2.525109052658081, + "logps/chosen": -201.82632446289062, + "logps/rejected": -233.27877807617188, + "loss": 1902.9713, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14684496819972992, + "rewards/margins": 0.11312825977802277, + "rewards/rejected": -0.2599732279777527, + "step": 9070 + }, + { + "epoch": 0.59, + "learning_rate": 2.1169523414197383e-06, + "logits/chosen": -2.182159662246704, + "logits/rejected": -2.1855149269104004, + "logps/chosen": -210.26461791992188, + "logps/rejected": -243.1604766845703, + "loss": 2167.5078, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.11458927392959595, + "rewards/margins": 0.095741868019104, + "rewards/rejected": -0.21033115684986115, + "step": 9080 + }, + { + "epoch": 0.59, + "learning_rate": 2.1113108563017267e-06, + "logits/chosen": -2.2826504707336426, + "logits/rejected": -2.0654351711273193, + "logps/chosen": -228.85623168945312, + "logps/rejected": -214.1038818359375, + "loss": 1925.9301, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16072991490364075, + "rewards/margins": 0.1587684154510498, + "rewards/rejected": -0.31949833035469055, + "step": 9090 + }, + { + "epoch": 0.6, + "learning_rate": 2.1056713987766905e-06, + "logits/chosen": -2.473038673400879, + "logits/rejected": -2.1360838413238525, + "logps/chosen": -229.9833526611328, + "logps/rejected": -195.61868286132812, + "loss": 1709.6678, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14783023297786713, + "rewards/margins": 0.14350786805152893, + "rewards/rejected": -0.29133808612823486, + "step": 9100 + }, + { + "epoch": 0.6, + "eval_logits/chosen": -2.3422412872314453, + "eval_logits/rejected": -2.155545234680176, + "eval_logps/chosen": -245.23779296875, + "eval_logps/rejected": -239.1156463623047, + "eval_loss": 1782.341796875, + "eval_rewards/accuracies": 0.6589999794960022, + "eval_rewards/chosen": -0.1323283314704895, + "eval_rewards/margins": 0.14270898699760437, + "eval_rewards/rejected": -0.27503734827041626, + "eval_runtime": 726.4537, + "eval_samples_per_second": 2.753, + "eval_steps_per_second": 1.377, + "step": 9100 + }, + { + "epoch": 0.6, + "learning_rate": 2.1000339982628022e-06, + "logits/chosen": -2.147550106048584, + "logits/rejected": -2.2366299629211426, + "logps/chosen": -266.4583740234375, + "logps/rejected": -243.4515838623047, + "loss": 1900.083, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.15185512602329254, + "rewards/margins": 0.11313346773386002, + "rewards/rejected": -0.26498860120773315, + "step": 9110 + }, + { + "epoch": 0.6, + "learning_rate": 2.0943986841675043e-06, + "logits/chosen": -2.3581080436706543, + "logits/rejected": -2.1352851390838623, + "logps/chosen": -214.7187042236328, + "logps/rejected": -210.26034545898438, + "loss": 1624.3727, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1150769367814064, + "rewards/margins": 0.14881470799446106, + "rewards/rejected": -0.26389163732528687, + "step": 9120 + }, + { + "epoch": 0.6, + "learning_rate": 2.088765485887356e-06, + "logits/chosen": -2.340144634246826, + "logits/rejected": -2.1474156379699707, + "logps/chosen": -257.22198486328125, + "logps/rejected": -228.04470825195312, + "loss": 1848.6959, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.12507091462612152, + "rewards/margins": 0.1244811862707138, + "rewards/rejected": -0.24955210089683533, + "step": 9130 + }, + { + "epoch": 0.6, + "learning_rate": 2.083134432807879e-06, + "logits/chosen": -2.263231039047241, + "logits/rejected": -2.2002289295196533, + "logps/chosen": -207.6758575439453, + "logps/rejected": -246.3755645751953, + "loss": 1857.1973, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1340373456478119, + "rewards/margins": 0.1718166023492813, + "rewards/rejected": -0.305853933095932, + "step": 9140 + }, + { + "epoch": 0.6, + "learning_rate": 2.077505554303404e-06, + "logits/chosen": -2.3470020294189453, + "logits/rejected": -2.311870813369751, + "logps/chosen": -184.0131072998047, + "logps/rejected": -202.27972412109375, + "loss": 1668.6959, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09461130201816559, + "rewards/margins": 0.13899998366832733, + "rewards/rejected": -0.2336113005876541, + "step": 9150 + }, + { + "epoch": 0.6, + "learning_rate": 2.071878879736918e-06, + "logits/chosen": -2.3313071727752686, + "logits/rejected": -2.1438777446746826, + "logps/chosen": -262.45343017578125, + "logps/rejected": -354.03533935546875, + "loss": 1832.6357, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15613846480846405, + "rewards/margins": 0.13298094272613525, + "rewards/rejected": -0.2891194224357605, + "step": 9160 + }, + { + "epoch": 0.6, + "learning_rate": 2.0662544384599136e-06, + "logits/chosen": -2.2310731410980225, + "logits/rejected": -2.1646816730499268, + "logps/chosen": -212.7046356201172, + "logps/rejected": -213.3751220703125, + "loss": 1754.624, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10598021745681763, + "rewards/margins": 0.17155803740024567, + "rewards/rejected": -0.2775382399559021, + "step": 9170 + }, + { + "epoch": 0.6, + "learning_rate": 2.0606322598122314e-06, + "logits/chosen": -2.2409818172454834, + "logits/rejected": -2.3552908897399902, + "logps/chosen": -204.3204345703125, + "logps/rejected": -232.7908935546875, + "loss": 2190.9982, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.17806866765022278, + "rewards/margins": 0.0880960077047348, + "rewards/rejected": -0.2661646902561188, + "step": 9180 + }, + { + "epoch": 0.6, + "learning_rate": 2.0550123731219085e-06, + "logits/chosen": -2.5051026344299316, + "logits/rejected": -2.342766284942627, + "logps/chosen": -265.1522216796875, + "logps/rejected": -239.1829071044922, + "loss": 1589.7291, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11972393840551376, + "rewards/margins": 0.15175560116767883, + "rewards/rejected": -0.2714795470237732, + "step": 9190 + }, + { + "epoch": 0.6, + "learning_rate": 2.0493948077050267e-06, + "logits/chosen": -2.1906094551086426, + "logits/rejected": -2.009308338165283, + "logps/chosen": -208.6791534423828, + "logps/rejected": -205.8546905517578, + "loss": 1829.7031, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1542314738035202, + "rewards/margins": 0.15265150368213654, + "rewards/rejected": -0.30688297748565674, + "step": 9200 + }, + { + "epoch": 0.6, + "eval_logits/chosen": -2.3467259407043457, + "eval_logits/rejected": -2.1592679023742676, + "eval_logps/chosen": -245.86827087402344, + "eval_logps/rejected": -240.8571319580078, + "eval_loss": 1774.454833984375, + "eval_rewards/accuracies": 0.6610000133514404, + "eval_rewards/chosen": -0.13863316178321838, + "eval_rewards/margins": 0.15381911396980286, + "eval_rewards/rejected": -0.29245227575302124, + "eval_runtime": 723.8065, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.382, + "step": 9200 + }, + { + "epoch": 0.6, + "learning_rate": 2.0437795928655596e-06, + "logits/chosen": -2.3635621070861816, + "logits/rejected": -2.3407657146453857, + "logps/chosen": -296.04876708984375, + "logps/rejected": -291.05877685546875, + "loss": 1739.4965, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1433614194393158, + "rewards/margins": 0.1319374144077301, + "rewards/rejected": -0.2752988338470459, + "step": 9210 + }, + { + "epoch": 0.6, + "learning_rate": 2.0381667578952184e-06, + "logits/chosen": -2.4439473152160645, + "logits/rejected": -2.2242536544799805, + "logps/chosen": -229.7306671142578, + "logps/rejected": -251.9036865234375, + "loss": 1798.8371, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.18857814371585846, + "rewards/margins": 0.1655600517988205, + "rewards/rejected": -0.35413819551467896, + "step": 9220 + }, + { + "epoch": 0.6, + "learning_rate": 2.0325563320732995e-06, + "logits/chosen": -2.5141654014587402, + "logits/rejected": -2.1215217113494873, + "logps/chosen": -277.11749267578125, + "logps/rejected": -250.9359893798828, + "loss": 1667.3076, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13779176771640778, + "rewards/margins": 0.17836323380470276, + "rewards/rejected": -0.31615501642227173, + "step": 9230 + }, + { + "epoch": 0.6, + "learning_rate": 2.026948344666532e-06, + "logits/chosen": -2.258357048034668, + "logits/rejected": -2.2154464721679688, + "logps/chosen": -216.21041870117188, + "logps/rejected": -231.43017578125, + "loss": 1620.7397, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1708449274301529, + "rewards/margins": 0.1566438525915146, + "rewards/rejected": -0.3274887800216675, + "step": 9240 + }, + { + "epoch": 0.61, + "learning_rate": 2.0213428249289257e-06, + "logits/chosen": -2.2519164085388184, + "logits/rejected": -2.161630153656006, + "logps/chosen": -209.21115112304688, + "logps/rejected": -229.04519653320312, + "loss": 1425.7735, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1067061796784401, + "rewards/margins": 0.1802651435136795, + "rewards/rejected": -0.2869713604450226, + "step": 9250 + }, + { + "epoch": 0.61, + "learning_rate": 2.0157398021016175e-06, + "logits/chosen": -2.2583141326904297, + "logits/rejected": -2.2162694931030273, + "logps/chosen": -161.82420349121094, + "logps/rejected": -218.95913696289062, + "loss": 2075.3656, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.14288866519927979, + "rewards/margins": 0.10930158197879791, + "rewards/rejected": -0.2521902620792389, + "step": 9260 + }, + { + "epoch": 0.61, + "learning_rate": 2.010139305412719e-06, + "logits/chosen": -2.508657455444336, + "logits/rejected": -2.2943289279937744, + "logps/chosen": -291.4534606933594, + "logps/rejected": -265.9161376953125, + "loss": 1707.8695, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14857791364192963, + "rewards/margins": 0.1682676374912262, + "rewards/rejected": -0.316845566034317, + "step": 9270 + }, + { + "epoch": 0.61, + "learning_rate": 2.0045413640771644e-06, + "logits/chosen": -2.2465755939483643, + "logits/rejected": -2.4008095264434814, + "logps/chosen": -269.99713134765625, + "logps/rejected": -290.6904602050781, + "loss": 1703.0684, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1483180820941925, + "rewards/margins": 0.1743774116039276, + "rewards/rejected": -0.32269546389579773, + "step": 9280 + }, + { + "epoch": 0.61, + "learning_rate": 1.998946007296558e-06, + "logits/chosen": -2.485224962234497, + "logits/rejected": -2.1421313285827637, + "logps/chosen": -324.39080810546875, + "logps/rejected": -276.89190673828125, + "loss": 1705.784, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.157902330160141, + "rewards/margins": 0.1593799889087677, + "rewards/rejected": -0.3172822892665863, + "step": 9290 + }, + { + "epoch": 0.61, + "learning_rate": 1.9933532642590215e-06, + "logits/chosen": -2.228764057159424, + "logits/rejected": -1.855963110923767, + "logps/chosen": -197.72039794921875, + "logps/rejected": -165.6393585205078, + "loss": 1540.8942, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08562113344669342, + "rewards/margins": 0.18079780042171478, + "rewards/rejected": -0.2664188742637634, + "step": 9300 + }, + { + "epoch": 0.61, + "eval_logits/chosen": -2.3514695167541504, + "eval_logits/rejected": -2.1634879112243652, + "eval_logps/chosen": -248.19981384277344, + "eval_logps/rejected": -243.58885192871094, + "eval_loss": 1782.968505859375, + "eval_rewards/accuracies": 0.6570000052452087, + "eval_rewards/chosen": -0.16194862127304077, + "eval_rewards/margins": 0.1578209102153778, + "eval_rewards/rejected": -0.3197695314884186, + "eval_runtime": 726.8341, + "eval_samples_per_second": 2.752, + "eval_steps_per_second": 1.376, + "step": 9300 + }, + { + "epoch": 0.61, + "learning_rate": 1.987763164139042e-06, + "logits/chosen": -2.383131742477417, + "logits/rejected": -2.182943105697632, + "logps/chosen": -220.27670288085938, + "logps/rejected": -235.8668670654297, + "loss": 1990.3887, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.15021634101867676, + "rewards/margins": 0.1333579570055008, + "rewards/rejected": -0.28357431292533875, + "step": 9310 + }, + { + "epoch": 0.61, + "learning_rate": 1.982175736097321e-06, + "logits/chosen": -2.0686111450195312, + "logits/rejected": -2.0829806327819824, + "logps/chosen": -297.60162353515625, + "logps/rejected": -323.6811218261719, + "loss": 2291.2773, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.23060569167137146, + "rewards/margins": 0.09059707075357437, + "rewards/rejected": -0.32120275497436523, + "step": 9320 + }, + { + "epoch": 0.61, + "learning_rate": 1.9765910092806196e-06, + "logits/chosen": -2.271430015563965, + "logits/rejected": -2.1739978790283203, + "logps/chosen": -190.2344970703125, + "logps/rejected": -182.0806884765625, + "loss": 1859.2246, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.12158989906311035, + "rewards/margins": 0.11504407227039337, + "rewards/rejected": -0.23663397133350372, + "step": 9330 + }, + { + "epoch": 0.61, + "learning_rate": 1.9710090128216083e-06, + "logits/chosen": -2.325185537338257, + "logits/rejected": -2.2322769165039062, + "logps/chosen": -241.9593505859375, + "logps/rejected": -244.9901123046875, + "loss": 1752.9023, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.19229312241077423, + "rewards/margins": 0.19193877279758453, + "rewards/rejected": -0.38423192501068115, + "step": 9340 + }, + { + "epoch": 0.61, + "learning_rate": 1.9654297758387155e-06, + "logits/chosen": -2.1726956367492676, + "logits/rejected": -2.1545259952545166, + "logps/chosen": -185.04246520996094, + "logps/rejected": -219.32510375976562, + "loss": 1810.7492, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.24172444641590118, + "rewards/margins": 0.13557168841362, + "rewards/rejected": -0.3772961497306824, + "step": 9350 + }, + { + "epoch": 0.61, + "learning_rate": 1.9598533274359736e-06, + "logits/chosen": -2.335035800933838, + "logits/rejected": -2.242264986038208, + "logps/chosen": -266.0108947753906, + "logps/rejected": -279.05572509765625, + "loss": 2427.3982, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2633058726787567, + "rewards/margins": 0.056832779198884964, + "rewards/rejected": -0.3201386630535126, + "step": 9360 + }, + { + "epoch": 0.61, + "learning_rate": 1.9542796967028697e-06, + "logits/chosen": -2.3382534980773926, + "logits/rejected": -2.2298545837402344, + "logps/chosen": -241.0985107421875, + "logps/rejected": -235.7290496826172, + "loss": 1784.1752, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.22857961058616638, + "rewards/margins": 0.12873327732086182, + "rewards/rejected": -0.3573128581047058, + "step": 9370 + }, + { + "epoch": 0.61, + "learning_rate": 1.948708912714192e-06, + "logits/chosen": -2.254430055618286, + "logits/rejected": -2.011399030685425, + "logps/chosen": -270.86187744140625, + "logps/rejected": -250.81558227539062, + "loss": 2033.2949, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.23916606605052948, + "rewards/margins": 0.10889426618814468, + "rewards/rejected": -0.3480603098869324, + "step": 9380 + }, + { + "epoch": 0.61, + "learning_rate": 1.9431410045298786e-06, + "logits/chosen": -2.1085152626037598, + "logits/rejected": -2.037166118621826, + "logps/chosen": -237.0220184326172, + "logps/rejected": -249.94583129882812, + "loss": 1782.4275, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.19896671175956726, + "rewards/margins": 0.13899557292461395, + "rewards/rejected": -0.33796226978302, + "step": 9390 + }, + { + "epoch": 0.62, + "learning_rate": 1.9375760011948654e-06, + "logits/chosen": -2.4289746284484863, + "logits/rejected": -2.2810416221618652, + "logps/chosen": -217.75021362304688, + "logps/rejected": -258.0467224121094, + "loss": 1477.422, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1807229220867157, + "rewards/margins": 0.17976048588752747, + "rewards/rejected": -0.36048340797424316, + "step": 9400 + }, + { + "epoch": 0.62, + "eval_logits/chosen": -2.345621109008789, + "eval_logits/rejected": -2.158308744430542, + "eval_logps/chosen": -252.24818420410156, + "eval_logps/rejected": -246.08737182617188, + "eval_loss": 1785.5960693359375, + "eval_rewards/accuracies": 0.6524999737739563, + "eval_rewards/chosen": -0.20243246853351593, + "eval_rewards/margins": 0.14232242107391357, + "eval_rewards/rejected": -0.3447548449039459, + "eval_runtime": 722.7813, + "eval_samples_per_second": 2.767, + "eval_steps_per_second": 1.384, + "step": 9400 + }, + { + "epoch": 0.62, + "learning_rate": 1.932013931738937e-06, + "logits/chosen": -2.327096462249756, + "logits/rejected": -2.1064553260803223, + "logps/chosen": -223.3422088623047, + "logps/rejected": -259.3307189941406, + "loss": 1314.2292, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.1974048614501953, + "rewards/margins": 0.23106737434864044, + "rewards/rejected": -0.42847222089767456, + "step": 9410 + }, + { + "epoch": 0.62, + "learning_rate": 1.9264548251765717e-06, + "logits/chosen": -2.4367785453796387, + "logits/rejected": -2.251880168914795, + "logps/chosen": -218.90817260742188, + "logps/rejected": -235.78543090820312, + "loss": 1347.5708, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15105506777763367, + "rewards/margins": 0.19989021122455597, + "rewards/rejected": -0.35094529390335083, + "step": 9420 + }, + { + "epoch": 0.62, + "learning_rate": 1.9208987105067924e-06, + "logits/chosen": -2.2413735389709473, + "logits/rejected": -2.1048600673675537, + "logps/chosen": -234.02755737304688, + "logps/rejected": -220.658935546875, + "loss": 2059.2354, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.19470515847206116, + "rewards/margins": 0.10519008338451385, + "rewards/rejected": -0.2998952269554138, + "step": 9430 + }, + { + "epoch": 0.62, + "learning_rate": 1.9153456167130154e-06, + "logits/chosen": -2.3501267433166504, + "logits/rejected": -2.341381311416626, + "logps/chosen": -225.01980590820312, + "logps/rejected": -263.4544372558594, + "loss": 1804.2916, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.20949192345142365, + "rewards/margins": 0.12235965579748154, + "rewards/rejected": -0.331851601600647, + "step": 9440 + }, + { + "epoch": 0.62, + "learning_rate": 1.9097955727628975e-06, + "logits/chosen": -2.3747715950012207, + "logits/rejected": -2.37276029586792, + "logps/chosen": -212.94384765625, + "logps/rejected": -238.12759399414062, + "loss": 1802.3164, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.16958339512348175, + "rewards/margins": 0.13248775899410248, + "rewards/rejected": -0.3020711839199066, + "step": 9450 + }, + { + "epoch": 0.62, + "learning_rate": 1.904248607608187e-06, + "logits/chosen": -2.284837007522583, + "logits/rejected": -2.315736770629883, + "logps/chosen": -272.3593444824219, + "logps/rejected": -240.7972869873047, + "loss": 1520.689, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15139371156692505, + "rewards/margins": 0.15544667840003967, + "rewards/rejected": -0.3068404197692871, + "step": 9460 + }, + { + "epoch": 0.62, + "learning_rate": 1.8987047501845714e-06, + "logits/chosen": -2.353510618209839, + "logits/rejected": -2.3260111808776855, + "logps/chosen": -183.2423858642578, + "logps/rejected": -195.4254608154297, + "loss": 1639.7561, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16317906975746155, + "rewards/margins": 0.18428602814674377, + "rewards/rejected": -0.3474651277065277, + "step": 9470 + }, + { + "epoch": 0.62, + "learning_rate": 1.8931640294115267e-06, + "logits/chosen": -2.1589865684509277, + "logits/rejected": -2.04636549949646, + "logps/chosen": -212.6373748779297, + "logps/rejected": -215.263671875, + "loss": 1550.8488, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.18631470203399658, + "rewards/margins": 0.18156476318836212, + "rewards/rejected": -0.3678794503211975, + "step": 9480 + }, + { + "epoch": 0.62, + "learning_rate": 1.8876264741921662e-06, + "logits/chosen": -2.121523380279541, + "logits/rejected": -2.143402576446533, + "logps/chosen": -208.8270721435547, + "logps/rejected": -221.06982421875, + "loss": 1578.1198, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1870156228542328, + "rewards/margins": 0.17678289115428925, + "rewards/rejected": -0.3637985587120056, + "step": 9490 + }, + { + "epoch": 0.62, + "learning_rate": 1.8820921134130912e-06, + "logits/chosen": -2.3421788215637207, + "logits/rejected": -1.9765698909759521, + "logps/chosen": -248.6728057861328, + "logps/rejected": -224.97183227539062, + "loss": 1495.3285, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.18160656094551086, + "rewards/margins": 0.21208277344703674, + "rewards/rejected": -0.3936893343925476, + "step": 9500 + }, + { + "epoch": 0.62, + "eval_logits/chosen": -2.3461034297943115, + "eval_logits/rejected": -2.1586921215057373, + "eval_logps/chosen": -253.2010498046875, + "eval_logps/rejected": -247.79739379882812, + "eval_loss": 1796.2745361328125, + "eval_rewards/accuracies": 0.6510000228881836, + "eval_rewards/chosen": -0.21196097135543823, + "eval_rewards/margins": 0.14989392459392548, + "eval_rewards/rejected": -0.3618549108505249, + "eval_runtime": 724.698, + "eval_samples_per_second": 2.76, + "eval_steps_per_second": 1.38, + "step": 9500 + }, + { + "epoch": 0.62, + "learning_rate": 1.8765609759442378e-06, + "logits/chosen": -2.2057645320892334, + "logits/rejected": -2.0962395668029785, + "logps/chosen": -263.9498291015625, + "logps/rejected": -265.99652099609375, + "loss": 1741.1148, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.22628538310527802, + "rewards/margins": 0.12858828902244568, + "rewards/rejected": -0.3548737168312073, + "step": 9510 + }, + { + "epoch": 0.62, + "learning_rate": 1.8710330906387288e-06, + "logits/chosen": -2.3963239192962646, + "logits/rejected": -2.344132900238037, + "logps/chosen": -263.30889892578125, + "logps/rejected": -298.1190185546875, + "loss": 1954.2197, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2482127696275711, + "rewards/margins": 0.11220197379589081, + "rewards/rejected": -0.3604147434234619, + "step": 9520 + }, + { + "epoch": 0.62, + "learning_rate": 1.8655084863327222e-06, + "logits/chosen": -2.3089566230773926, + "logits/rejected": -2.33139705657959, + "logps/chosen": -202.14541625976562, + "logps/rejected": -216.45089721679688, + "loss": 2188.0619, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.1868743598461151, + "rewards/margins": 0.08143356442451477, + "rewards/rejected": -0.26830795407295227, + "step": 9530 + }, + { + "epoch": 0.62, + "learning_rate": 1.8599871918452603e-06, + "logits/chosen": -2.1625704765319824, + "logits/rejected": -2.15848970413208, + "logps/chosen": -240.3632049560547, + "logps/rejected": -269.21112060546875, + "loss": 1604.9603, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.17271386086940765, + "rewards/margins": 0.155984565615654, + "rewards/rejected": -0.32869842648506165, + "step": 9540 + }, + { + "epoch": 0.62, + "learning_rate": 1.8544692359781192e-06, + "logits/chosen": -2.36810040473938, + "logits/rejected": -2.133775234222412, + "logps/chosen": -202.4742889404297, + "logps/rejected": -189.27862548828125, + "loss": 1458.3448, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14889639616012573, + "rewards/margins": 0.17751574516296387, + "rewards/rejected": -0.3264121413230896, + "step": 9550 + }, + { + "epoch": 0.63, + "learning_rate": 1.8489546475156602e-06, + "logits/chosen": -2.518655300140381, + "logits/rejected": -2.2955431938171387, + "logps/chosen": -238.36276245117188, + "logps/rejected": -234.1089630126953, + "loss": 1833.2234, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.18869927525520325, + "rewards/margins": 0.13914178311824799, + "rewards/rejected": -0.32784101366996765, + "step": 9560 + }, + { + "epoch": 0.63, + "learning_rate": 1.8434434552246778e-06, + "logits/chosen": -2.1662681102752686, + "logits/rejected": -2.0950560569763184, + "logps/chosen": -221.53689575195312, + "logps/rejected": -230.15853881835938, + "loss": 1530.9552, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11905352771282196, + "rewards/margins": 0.19884634017944336, + "rewards/rejected": -0.3178998529911041, + "step": 9570 + }, + { + "epoch": 0.63, + "learning_rate": 1.837935687854251e-06, + "logits/chosen": -2.3633503913879395, + "logits/rejected": -2.1214184761047363, + "logps/chosen": -236.35519409179688, + "logps/rejected": -226.7471466064453, + "loss": 1417.6899, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.14808468520641327, + "rewards/margins": 0.2219206541776657, + "rewards/rejected": -0.3700053095817566, + "step": 9580 + }, + { + "epoch": 0.63, + "learning_rate": 1.832431374135592e-06, + "logits/chosen": -2.4873459339141846, + "logits/rejected": -2.0834736824035645, + "logps/chosen": -266.6532287597656, + "logps/rejected": -264.7237243652344, + "loss": 1751.1215, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1821785420179367, + "rewards/margins": 0.1694364994764328, + "rewards/rejected": -0.3516150414943695, + "step": 9590 + }, + { + "epoch": 0.63, + "learning_rate": 1.8269305427818977e-06, + "logits/chosen": -2.4908695220947266, + "logits/rejected": -2.3433737754821777, + "logps/chosen": -222.4200439453125, + "logps/rejected": -211.399658203125, + "loss": 1647.9816, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13803806900978088, + "rewards/margins": 0.16643409430980682, + "rewards/rejected": -0.3044721782207489, + "step": 9600 + }, + { + "epoch": 0.63, + "eval_logits/chosen": -2.353161573410034, + "eval_logits/rejected": -2.165470600128174, + "eval_logps/chosen": -248.5635223388672, + "eval_logps/rejected": -244.8505401611328, + "eval_loss": 1805.82275390625, + "eval_rewards/accuracies": 0.6539999842643738, + "eval_rewards/chosen": -0.1655854731798172, + "eval_rewards/margins": 0.16680093109607697, + "eval_rewards/rejected": -0.33238640427589417, + "eval_runtime": 724.1311, + "eval_samples_per_second": 2.762, + "eval_steps_per_second": 1.381, + "step": 9600 + }, + { + "epoch": 0.63, + "learning_rate": 1.821433222488199e-06, + "logits/chosen": -2.3555164337158203, + "logits/rejected": -2.039760112762451, + "logps/chosen": -238.9461212158203, + "logps/rejected": -227.8724365234375, + "loss": 1930.6977, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1333135962486267, + "rewards/margins": 0.166739359498024, + "rewards/rejected": -0.3000529408454895, + "step": 9610 + }, + { + "epoch": 0.63, + "learning_rate": 1.8159394419312112e-06, + "logits/chosen": -2.3798041343688965, + "logits/rejected": -2.197470188140869, + "logps/chosen": -272.6890869140625, + "logps/rejected": -249.67861938476562, + "loss": 1383.2759, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.1395907700061798, + "rewards/margins": 0.23528221249580383, + "rewards/rejected": -0.37487298250198364, + "step": 9620 + }, + { + "epoch": 0.63, + "learning_rate": 1.8104492297691845e-06, + "logits/chosen": -2.3357458114624023, + "logits/rejected": -2.1194615364074707, + "logps/chosen": -244.08535766601562, + "logps/rejected": -236.9788360595703, + "loss": 1766.9283, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1756221503019333, + "rewards/margins": 0.17006057500839233, + "rewards/rejected": -0.3456827402114868, + "step": 9630 + }, + { + "epoch": 0.63, + "learning_rate": 1.8049626146417562e-06, + "logits/chosen": -2.1383423805236816, + "logits/rejected": -1.9963083267211914, + "logps/chosen": -169.82345581054688, + "logps/rejected": -178.7533721923828, + "loss": 2164.8064, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.14747557044029236, + "rewards/margins": 0.1264285296201706, + "rewards/rejected": -0.27390408515930176, + "step": 9640 + }, + { + "epoch": 0.63, + "learning_rate": 1.7994796251697983e-06, + "logits/chosen": -2.241729497909546, + "logits/rejected": -2.105792284011841, + "logps/chosen": -210.55819702148438, + "logps/rejected": -267.66064453125, + "loss": 1514.0446, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1851930171251297, + "rewards/margins": 0.17537666857242584, + "rewards/rejected": -0.36056965589523315, + "step": 9650 + }, + { + "epoch": 0.63, + "learning_rate": 1.794000289955269e-06, + "logits/chosen": -2.283034563064575, + "logits/rejected": -2.115142345428467, + "logps/chosen": -287.27728271484375, + "logps/rejected": -270.07666015625, + "loss": 2013.5223, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.20217004418373108, + "rewards/margins": 0.14246216416358948, + "rewards/rejected": -0.34463220834732056, + "step": 9660 + }, + { + "epoch": 0.63, + "learning_rate": 1.7885246375810646e-06, + "logits/chosen": -2.2674050331115723, + "logits/rejected": -1.9900856018066406, + "logps/chosen": -237.94100952148438, + "logps/rejected": -250.2483367919922, + "loss": 1496.8723, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13667115569114685, + "rewards/margins": 0.1606934368610382, + "rewards/rejected": -0.29736456274986267, + "step": 9670 + }, + { + "epoch": 0.63, + "learning_rate": 1.7830526966108713e-06, + "logits/chosen": -2.1287639141082764, + "logits/rejected": -1.9397249221801758, + "logps/chosen": -206.8074188232422, + "logps/rejected": -198.19400024414062, + "loss": 1426.6112, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2033451497554779, + "rewards/margins": 0.19998101890087128, + "rewards/rejected": -0.4033261835575104, + "step": 9680 + }, + { + "epoch": 0.63, + "learning_rate": 1.7775844955890129e-06, + "logits/chosen": -2.282334089279175, + "logits/rejected": -2.120537757873535, + "logps/chosen": -224.4355010986328, + "logps/rejected": -235.31326293945312, + "loss": 1387.2142, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12889166176319122, + "rewards/margins": 0.22454425692558289, + "rewards/rejected": -0.3534359335899353, + "step": 9690 + }, + { + "epoch": 0.63, + "learning_rate": 1.7721200630403046e-06, + "logits/chosen": -2.3532631397247314, + "logits/rejected": -2.1655144691467285, + "logps/chosen": -212.76953125, + "logps/rejected": -249.0178680419922, + "loss": 2050.0473, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.17007184028625488, + "rewards/margins": 0.10702402889728546, + "rewards/rejected": -0.27709585428237915, + "step": 9700 + }, + { + "epoch": 0.63, + "eval_logits/chosen": -2.3500943183898926, + "eval_logits/rejected": -2.1625893115997314, + "eval_logps/chosen": -249.26937866210938, + "eval_logps/rejected": -244.1856231689453, + "eval_loss": 1781.771728515625, + "eval_rewards/accuracies": 0.6585000157356262, + "eval_rewards/chosen": -0.17264403402805328, + "eval_rewards/margins": 0.15309324860572815, + "eval_rewards/rejected": -0.32573723793029785, + "eval_runtime": 725.1374, + "eval_samples_per_second": 2.758, + "eval_steps_per_second": 1.379, + "step": 9700 + }, + { + "epoch": 0.64, + "learning_rate": 1.7666594274699037e-06, + "logits/chosen": -2.2733945846557617, + "logits/rejected": -2.125950336456299, + "logps/chosen": -268.30718994140625, + "logps/rejected": -250.86489868164062, + "loss": 1467.0482, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15616917610168457, + "rewards/margins": 0.20761807262897491, + "rewards/rejected": -0.3637872636318207, + "step": 9710 + }, + { + "epoch": 0.64, + "learning_rate": 1.76120261736316e-06, + "logits/chosen": -2.3260655403137207, + "logits/rejected": -1.9596723318099976, + "logps/chosen": -245.9228057861328, + "logps/rejected": -236.8369598388672, + "loss": 1604.5218, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.19207733869552612, + "rewards/margins": 0.1805742084980011, + "rewards/rejected": -0.37265151739120483, + "step": 9720 + }, + { + "epoch": 0.64, + "learning_rate": 1.755749661185468e-06, + "logits/chosen": -2.3855628967285156, + "logits/rejected": -1.958176851272583, + "logps/chosen": -308.1893615722656, + "logps/rejected": -268.2945251464844, + "loss": 1685.2773, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13739819824695587, + "rewards/margins": 0.1893884837627411, + "rewards/rejected": -0.32678669691085815, + "step": 9730 + }, + { + "epoch": 0.64, + "learning_rate": 1.7503005873821183e-06, + "logits/chosen": -2.33542799949646, + "logits/rejected": -2.2756717205047607, + "logps/chosen": -175.3501434326172, + "logps/rejected": -215.01730346679688, + "loss": 1826.0061, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1492181122303009, + "rewards/margins": 0.13403229415416718, + "rewards/rejected": -0.2832503914833069, + "step": 9740 + }, + { + "epoch": 0.64, + "learning_rate": 1.744855424378148e-06, + "logits/chosen": -2.1913769245147705, + "logits/rejected": -2.2056326866149902, + "logps/chosen": -199.3675994873047, + "logps/rejected": -243.65487670898438, + "loss": 1348.4872, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10740645229816437, + "rewards/margins": 0.20841753482818604, + "rewards/rejected": -0.3158240020275116, + "step": 9750 + }, + { + "epoch": 0.64, + "learning_rate": 1.7394142005781973e-06, + "logits/chosen": -2.1301426887512207, + "logits/rejected": -2.220323085784912, + "logps/chosen": -277.7008972167969, + "logps/rejected": -299.0531005859375, + "loss": 1575.9425, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.159898579120636, + "rewards/margins": 0.1778123378753662, + "rewards/rejected": -0.3377109467983246, + "step": 9760 + }, + { + "epoch": 0.64, + "learning_rate": 1.7339769443663528e-06, + "logits/chosen": -2.3279261589050293, + "logits/rejected": -2.1848256587982178, + "logps/chosen": -161.63812255859375, + "logps/rejected": -174.63168334960938, + "loss": 1832.999, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1869591474533081, + "rewards/margins": 0.12290282547473907, + "rewards/rejected": -0.309861958026886, + "step": 9770 + }, + { + "epoch": 0.64, + "learning_rate": 1.7285436841060078e-06, + "logits/chosen": -2.510305404663086, + "logits/rejected": -2.224672317504883, + "logps/chosen": -288.49371337890625, + "logps/rejected": -264.77435302734375, + "loss": 1402.2434, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12365376949310303, + "rewards/margins": 0.19234833121299744, + "rewards/rejected": -0.31600213050842285, + "step": 9780 + }, + { + "epoch": 0.64, + "learning_rate": 1.7231144481397083e-06, + "logits/chosen": -2.3978075981140137, + "logits/rejected": -2.3237767219543457, + "logps/chosen": -233.5013427734375, + "logps/rejected": -224.5458984375, + "loss": 1697.8324, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13102979958057404, + "rewards/margins": 0.16483241319656372, + "rewards/rejected": -0.29586222767829895, + "step": 9790 + }, + { + "epoch": 0.64, + "learning_rate": 1.7176892647890092e-06, + "logits/chosen": -2.4705073833465576, + "logits/rejected": -2.2312042713165283, + "logps/chosen": -252.5708770751953, + "logps/rejected": -227.5338134765625, + "loss": 1848.6725, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16904866695404053, + "rewards/margins": 0.13844363391399384, + "rewards/rejected": -0.30749231576919556, + "step": 9800 + }, + { + "epoch": 0.64, + "eval_logits/chosen": -2.343329668045044, + "eval_logits/rejected": -2.156418561935425, + "eval_logps/chosen": -247.46359252929688, + "eval_logps/rejected": -243.70355224609375, + "eval_loss": 1796.389404296875, + "eval_rewards/accuracies": 0.6610000133514404, + "eval_rewards/chosen": -0.1545863300561905, + "eval_rewards/margins": 0.16632995009422302, + "eval_rewards/rejected": -0.3209163248538971, + "eval_runtime": 726.3607, + "eval_samples_per_second": 2.753, + "eval_steps_per_second": 1.377, + "step": 9800 + }, + { + "epoch": 0.64, + "learning_rate": 1.7122681623543239e-06, + "logits/chosen": -2.4738147258758545, + "logits/rejected": -2.210662364959717, + "logps/chosen": -261.3623962402344, + "logps/rejected": -269.6300048828125, + "loss": 1656.6836, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14373771846294403, + "rewards/margins": 0.194814994931221, + "rewards/rejected": -0.3385527431964874, + "step": 9810 + }, + { + "epoch": 0.64, + "learning_rate": 1.7068511691147788e-06, + "logits/chosen": -2.2168641090393066, + "logits/rejected": -2.2618637084960938, + "logps/chosen": -205.35595703125, + "logps/rejected": -233.04379272460938, + "loss": 1725.5117, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11224903911352158, + "rewards/margins": 0.19318260252475739, + "rewards/rejected": -0.30543166399002075, + "step": 9820 + }, + { + "epoch": 0.64, + "learning_rate": 1.7014383133280636e-06, + "logits/chosen": -2.452572822570801, + "logits/rejected": -2.090609073638916, + "logps/chosen": -269.6567687988281, + "logps/rejected": -226.94692993164062, + "loss": 2184.6486, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.21022340655326843, + "rewards/margins": 0.10684118419885635, + "rewards/rejected": -0.3170645833015442, + "step": 9830 + }, + { + "epoch": 0.64, + "learning_rate": 1.696029623230286e-06, + "logits/chosen": -2.4171395301818848, + "logits/rejected": -2.3338518142700195, + "logps/chosen": -270.8750305175781, + "logps/rejected": -304.5694274902344, + "loss": 1857.9078, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.14952710270881653, + "rewards/margins": 0.14915330708026886, + "rewards/rejected": -0.2986803948879242, + "step": 9840 + }, + { + "epoch": 0.64, + "learning_rate": 1.6906251270358229e-06, + "logits/chosen": -2.4054393768310547, + "logits/rejected": -2.258305788040161, + "logps/chosen": -274.5176086425781, + "logps/rejected": -240.35842895507812, + "loss": 1680.9318, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11663287878036499, + "rewards/margins": 0.1558263897895813, + "rewards/rejected": -0.2724592983722687, + "step": 9850 + }, + { + "epoch": 0.65, + "learning_rate": 1.685224852937174e-06, + "logits/chosen": -2.216269016265869, + "logits/rejected": -2.1068036556243896, + "logps/chosen": -205.4796600341797, + "logps/rejected": -273.53826904296875, + "loss": 1176.6585, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.07114885747432709, + "rewards/margins": 0.26199641823768616, + "rewards/rejected": -0.33314526081085205, + "step": 9860 + }, + { + "epoch": 0.65, + "learning_rate": 1.6798288291048136e-06, + "logits/chosen": -2.1753268241882324, + "logits/rejected": -2.0786020755767822, + "logps/chosen": -233.43582153320312, + "logps/rejected": -225.013671875, + "loss": 1643.7133, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10194216668605804, + "rewards/margins": 0.2028282880783081, + "rewards/rejected": -0.30477046966552734, + "step": 9870 + }, + { + "epoch": 0.65, + "learning_rate": 1.6744370836870466e-06, + "logits/chosen": -2.5711913108825684, + "logits/rejected": -2.2863872051239014, + "logps/chosen": -343.8927307128906, + "logps/rejected": -280.05975341796875, + "loss": 1962.7498, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.09199172258377075, + "rewards/margins": 0.19684836268424988, + "rewards/rejected": -0.288840115070343, + "step": 9880 + }, + { + "epoch": 0.65, + "learning_rate": 1.6690496448098576e-06, + "logits/chosen": -2.2480640411376953, + "logits/rejected": -1.9541877508163452, + "logps/chosen": -242.25094604492188, + "logps/rejected": -237.30178833007812, + "loss": 1781.4516, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1321474015712738, + "rewards/margins": 0.15909911692142487, + "rewards/rejected": -0.2912465035915375, + "step": 9890 + }, + { + "epoch": 0.65, + "learning_rate": 1.6636665405767666e-06, + "logits/chosen": -2.3545076847076416, + "logits/rejected": -2.1871304512023926, + "logps/chosen": -246.6212921142578, + "logps/rejected": -243.40444946289062, + "loss": 1784.2059, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0885036438703537, + "rewards/margins": 0.15673181414604187, + "rewards/rejected": -0.24523547291755676, + "step": 9900 + }, + { + "epoch": 0.65, + "eval_logits/chosen": -2.3468716144561768, + "eval_logits/rejected": -2.1599361896514893, + "eval_logps/chosen": -245.296630859375, + "eval_logps/rejected": -240.30271911621094, + "eval_loss": 1775.6644287109375, + "eval_rewards/accuracies": 0.6625000238418579, + "eval_rewards/chosen": -0.13291673362255096, + "eval_rewards/margins": 0.1539914458990097, + "eval_rewards/rejected": -0.28690820932388306, + "eval_runtime": 725.6555, + "eval_samples_per_second": 2.756, + "eval_steps_per_second": 1.378, + "step": 9900 + }, + { + "epoch": 0.65, + "learning_rate": 1.6582877990686827e-06, + "logits/chosen": -2.3545796871185303, + "logits/rejected": -2.2970447540283203, + "logps/chosen": -134.80886840820312, + "logps/rejected": -172.93389892578125, + "loss": 1614.8482, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.13514499366283417, + "rewards/margins": 0.1579369157552719, + "rewards/rejected": -0.2930819094181061, + "step": 9910 + }, + { + "epoch": 0.65, + "learning_rate": 1.6529134483437562e-06, + "logits/chosen": -2.3577752113342285, + "logits/rejected": -2.1761374473571777, + "logps/chosen": -214.59286499023438, + "logps/rejected": -197.01571655273438, + "loss": 1277.2855, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0901859924197197, + "rewards/margins": 0.21876561641693115, + "rewards/rejected": -0.30895155668258667, + "step": 9920 + }, + { + "epoch": 0.65, + "learning_rate": 1.647543516437233e-06, + "logits/chosen": -2.3127143383026123, + "logits/rejected": -2.252162456512451, + "logps/chosen": -214.50137329101562, + "logps/rejected": -246.9535675048828, + "loss": 2047.5131, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.1546315997838974, + "rewards/margins": 0.117673359811306, + "rewards/rejected": -0.2723049521446228, + "step": 9930 + }, + { + "epoch": 0.65, + "learning_rate": 1.6421780313613088e-06, + "logits/chosen": -2.4648447036743164, + "logits/rejected": -2.0659372806549072, + "logps/chosen": -217.0897216796875, + "logps/rejected": -202.35964965820312, + "loss": 1451.7236, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12348121404647827, + "rewards/margins": 0.18459756672382355, + "rewards/rejected": -0.308078795671463, + "step": 9940 + }, + { + "epoch": 0.65, + "learning_rate": 1.6368170211049816e-06, + "logits/chosen": -2.371119737625122, + "logits/rejected": -1.8730380535125732, + "logps/chosen": -300.0697326660156, + "logps/rejected": -254.29638671875, + "loss": 2004.6252, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.16269531846046448, + "rewards/margins": 0.14338134229183197, + "rewards/rejected": -0.30607667565345764, + "step": 9950 + }, + { + "epoch": 0.65, + "learning_rate": 1.6314605136339074e-06, + "logits/chosen": -2.372769594192505, + "logits/rejected": -2.2136123180389404, + "logps/chosen": -213.3172149658203, + "logps/rejected": -205.68411254882812, + "loss": 2142.3727, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1773051619529724, + "rewards/margins": 0.10266335308551788, + "rewards/rejected": -0.2799685001373291, + "step": 9960 + }, + { + "epoch": 0.65, + "learning_rate": 1.6261085368902526e-06, + "logits/chosen": -2.5684804916381836, + "logits/rejected": -2.2169997692108154, + "logps/chosen": -282.66668701171875, + "logps/rejected": -249.35928344726562, + "loss": 2015.8521, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17725810408592224, + "rewards/margins": 0.1032206267118454, + "rewards/rejected": -0.28047871589660645, + "step": 9970 + }, + { + "epoch": 0.65, + "learning_rate": 1.6207611187925503e-06, + "logits/chosen": -2.2575650215148926, + "logits/rejected": -2.2796449661254883, + "logps/chosen": -221.99356079101562, + "logps/rejected": -285.18817138671875, + "loss": 1402.7092, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1316666156053543, + "rewards/margins": 0.16876843571662903, + "rewards/rejected": -0.30043506622314453, + "step": 9980 + }, + { + "epoch": 0.65, + "learning_rate": 1.6154182872355512e-06, + "logits/chosen": -2.3123984336853027, + "logits/rejected": -2.365647554397583, + "logps/chosen": -179.47439575195312, + "logps/rejected": -205.01754760742188, + "loss": 2203.9023, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.15656927227973938, + "rewards/margins": 0.11123538017272949, + "rewards/rejected": -0.26780465245246887, + "step": 9990 + }, + { + "epoch": 0.65, + "learning_rate": 1.610080070090084e-06, + "logits/chosen": -2.3012657165527344, + "logits/rejected": -2.237123489379883, + "logps/chosen": -195.8773193359375, + "logps/rejected": -203.55047607421875, + "loss": 1470.263, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.14956019818782806, + "rewards/margins": 0.16701506078243256, + "rewards/rejected": -0.3165753185749054, + "step": 10000 + }, + { + "epoch": 0.65, + "eval_logits/chosen": -2.3442938327789307, + "eval_logits/rejected": -2.1574673652648926, + "eval_logps/chosen": -245.648681640625, + "eval_logps/rejected": -240.724365234375, + "eval_loss": 1772.5384521484375, + "eval_rewards/accuracies": 0.6579999923706055, + "eval_rewards/chosen": -0.136437326669693, + "eval_rewards/margins": 0.15468725562095642, + "eval_rewards/rejected": -0.2911245822906494, + "eval_runtime": 725.8522, + "eval_samples_per_second": 2.755, + "eval_steps_per_second": 1.378, + "step": 10000 + }, + { + "epoch": 0.65, + "learning_rate": 1.6047464952029034e-06, + "logits/chosen": -2.4696366786956787, + "logits/rejected": -2.3387560844421387, + "logps/chosen": -264.2880554199219, + "logps/rejected": -293.85455322265625, + "loss": 1207.3434, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.10000646114349365, + "rewards/margins": 0.21340465545654297, + "rewards/rejected": -0.31341108679771423, + "step": 10010 + }, + { + "epoch": 0.66, + "learning_rate": 1.5994175903965486e-06, + "logits/chosen": -2.204178810119629, + "logits/rejected": -2.0907106399536133, + "logps/chosen": -264.9002685546875, + "logps/rejected": -285.266357421875, + "loss": 1572.2071, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14569909870624542, + "rewards/margins": 0.18611545860767365, + "rewards/rejected": -0.33181455731391907, + "step": 10020 + }, + { + "epoch": 0.66, + "learning_rate": 1.5940933834691977e-06, + "logits/chosen": -2.6393990516662598, + "logits/rejected": -1.9954341650009155, + "logps/chosen": -318.2119445800781, + "logps/rejected": -229.7041015625, + "loss": 1494.415, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.12946000695228577, + "rewards/margins": 0.1938951015472412, + "rewards/rejected": -0.32335513830184937, + "step": 10030 + }, + { + "epoch": 0.66, + "learning_rate": 1.588773902194522e-06, + "logits/chosen": -2.153494358062744, + "logits/rejected": -1.8746439218521118, + "logps/chosen": -221.29696655273438, + "logps/rejected": -243.8973388671875, + "loss": 2618.1234, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1721726804971695, + "rewards/margins": 0.2684074342250824, + "rewards/rejected": -0.4405801296234131, + "step": 10040 + }, + { + "epoch": 0.66, + "learning_rate": 1.583459174321541e-06, + "logits/chosen": -2.1277883052825928, + "logits/rejected": -1.9743620157241821, + "logps/chosen": -226.68612670898438, + "logps/rejected": -216.8396453857422, + "loss": 1734.6359, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17068837583065033, + "rewards/margins": 0.17247551679611206, + "rewards/rejected": -0.3431639075279236, + "step": 10050 + }, + { + "epoch": 0.66, + "learning_rate": 1.5781492275744797e-06, + "logits/chosen": -2.529752492904663, + "logits/rejected": -2.1446166038513184, + "logps/chosen": -305.91815185546875, + "logps/rejected": -305.4971618652344, + "loss": 1542.6445, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.11673681437969208, + "rewards/margins": 0.25796908140182495, + "rewards/rejected": -0.3747059106826782, + "step": 10060 + }, + { + "epoch": 0.66, + "learning_rate": 1.5728440896526215e-06, + "logits/chosen": -2.2576258182525635, + "logits/rejected": -2.0778629779815674, + "logps/chosen": -286.76837158203125, + "logps/rejected": -255.7520751953125, + "loss": 1687.0375, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13366422057151794, + "rewards/margins": 0.151865154504776, + "rewards/rejected": -0.28552937507629395, + "step": 10070 + }, + { + "epoch": 0.66, + "learning_rate": 1.5675437882301633e-06, + "logits/chosen": -2.3388214111328125, + "logits/rejected": -2.1428470611572266, + "logps/chosen": -230.275146484375, + "logps/rejected": -207.76254272460938, + "loss": 1935.9998, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11230950057506561, + "rewards/margins": 0.12283160537481308, + "rewards/rejected": -0.2351410835981369, + "step": 10080 + }, + { + "epoch": 0.66, + "learning_rate": 1.5622483509560748e-06, + "logits/chosen": -2.2283027172088623, + "logits/rejected": -2.2405989170074463, + "logps/chosen": -184.2806854248047, + "logps/rejected": -231.23104858398438, + "loss": 1743.1004, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1355154663324356, + "rewards/margins": 0.16768063604831696, + "rewards/rejected": -0.30319613218307495, + "step": 10090 + }, + { + "epoch": 0.66, + "learning_rate": 1.5569578054539506e-06, + "logits/chosen": -2.279250383377075, + "logits/rejected": -1.950156569480896, + "logps/chosen": -290.9871520996094, + "logps/rejected": -232.88156127929688, + "loss": 1144.0941, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.11790060997009277, + "rewards/margins": 0.25757458806037903, + "rewards/rejected": -0.3754751682281494, + "step": 10100 + }, + { + "epoch": 0.66, + "eval_logits/chosen": -2.339332103729248, + "eval_logits/rejected": -2.1528501510620117, + "eval_logps/chosen": -244.95468139648438, + "eval_logps/rejected": -240.91522216796875, + "eval_loss": 1776.5482177734375, + "eval_rewards/accuracies": 0.6614999771118164, + "eval_rewards/chosen": -0.12949733436107635, + "eval_rewards/margins": 0.1635356992483139, + "eval_rewards/rejected": -0.29303303360939026, + "eval_runtime": 725.4527, + "eval_samples_per_second": 2.757, + "eval_steps_per_second": 1.378, + "step": 10100 + }, + { + "epoch": 0.66, + "learning_rate": 1.551672179321867e-06, + "logits/chosen": -2.2603707313537598, + "logits/rejected": -2.33063006401062, + "logps/chosen": -225.4585723876953, + "logps/rejected": -221.43777465820312, + "loss": 1686.2473, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09601660072803497, + "rewards/margins": 0.17150314152240753, + "rewards/rejected": -0.2675197124481201, + "step": 10110 + }, + { + "epoch": 0.66, + "learning_rate": 1.5463915001322398e-06, + "logits/chosen": -2.2999863624572754, + "logits/rejected": -2.158628463745117, + "logps/chosen": -259.41180419921875, + "logps/rejected": -259.23175048828125, + "loss": 2175.3219, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13755428791046143, + "rewards/margins": 0.14187631011009216, + "rewards/rejected": -0.279430627822876, + "step": 10120 + }, + { + "epoch": 0.66, + "learning_rate": 1.5411157954316784e-06, + "logits/chosen": -2.2974162101745605, + "logits/rejected": -2.2118053436279297, + "logps/chosen": -212.8948211669922, + "logps/rejected": -214.84414672851562, + "loss": 1701.4238, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11017285287380219, + "rewards/margins": 0.15639138221740723, + "rewards/rejected": -0.2665642201900482, + "step": 10130 + }, + { + "epoch": 0.66, + "learning_rate": 1.535845092740843e-06, + "logits/chosen": -2.470548629760742, + "logits/rejected": -2.306570053100586, + "logps/chosen": -249.66329956054688, + "logps/rejected": -272.49053955078125, + "loss": 2000.9893, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12342099845409393, + "rewards/margins": 0.10431381314992905, + "rewards/rejected": -0.22773483395576477, + "step": 10140 + }, + { + "epoch": 0.66, + "learning_rate": 1.5305794195543005e-06, + "logits/chosen": -2.3118038177490234, + "logits/rejected": -2.3743338584899902, + "logps/chosen": -220.2370147705078, + "logps/rejected": -222.10546875, + "loss": 1502.9931, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1305791139602661, + "rewards/margins": 0.1879984438419342, + "rewards/rejected": -0.3185775578022003, + "step": 10150 + }, + { + "epoch": 0.66, + "learning_rate": 1.5253188033403816e-06, + "logits/chosen": -2.330814838409424, + "logits/rejected": -2.433004379272461, + "logps/chosen": -186.08499145507812, + "logps/rejected": -211.61630249023438, + "loss": 2086.2031, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.15350119769573212, + "rewards/margins": 0.08872922509908676, + "rewards/rejected": -0.24223044514656067, + "step": 10160 + }, + { + "epoch": 0.67, + "learning_rate": 1.520063271541037e-06, + "logits/chosen": -2.319892406463623, + "logits/rejected": -2.222487688064575, + "logps/chosen": -198.03294372558594, + "logps/rejected": -202.96609497070312, + "loss": 1855.0166, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.19528180360794067, + "rewards/margins": 0.1623615324497223, + "rewards/rejected": -0.35764333605766296, + "step": 10170 + }, + { + "epoch": 0.67, + "learning_rate": 1.5148128515716954e-06, + "logits/chosen": -2.5416927337646484, + "logits/rejected": -1.908489465713501, + "logps/chosen": -280.6117248535156, + "logps/rejected": -228.46212768554688, + "loss": 1646.4756, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.14082498848438263, + "rewards/margins": 0.20224598050117493, + "rewards/rejected": -0.343070924282074, + "step": 10180 + }, + { + "epoch": 0.67, + "learning_rate": 1.5095675708211197e-06, + "logits/chosen": -2.362449884414673, + "logits/rejected": -2.357105016708374, + "logps/chosen": -216.79443359375, + "logps/rejected": -246.13461303710938, + "loss": 2490.6564, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.25633513927459717, + "rewards/margins": 0.05996602773666382, + "rewards/rejected": -0.316301167011261, + "step": 10190 + }, + { + "epoch": 0.67, + "learning_rate": 1.504327456651263e-06, + "logits/chosen": -2.305445432662964, + "logits/rejected": -2.234525203704834, + "logps/chosen": -286.28363037109375, + "logps/rejected": -270.63360595703125, + "loss": 1890.1879, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.24880632758140564, + "rewards/margins": 0.11314669996500015, + "rewards/rejected": -0.3619530200958252, + "step": 10200 + }, + { + "epoch": 0.67, + "eval_logits/chosen": -2.3296923637390137, + "eval_logits/rejected": -2.144120454788208, + "eval_logps/chosen": -250.76393127441406, + "eval_logps/rejected": -244.91712951660156, + "eval_loss": 1785.0318603515625, + "eval_rewards/accuracies": 0.652999997138977, + "eval_rewards/chosen": -0.1875898540019989, + "eval_rewards/margins": 0.14546243846416473, + "eval_rewards/rejected": -0.33305224776268005, + "eval_runtime": 722.9768, + "eval_samples_per_second": 2.766, + "eval_steps_per_second": 1.383, + "step": 10200 + }, + { + "epoch": 0.67, + "learning_rate": 1.4990925363971284e-06, + "logits/chosen": -2.388564348220825, + "logits/rejected": -1.984683632850647, + "logps/chosen": -311.1983642578125, + "logps/rejected": -266.94000244140625, + "loss": 1314.9645, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21275189518928528, + "rewards/margins": 0.24230685830116272, + "rewards/rejected": -0.4550587236881256, + "step": 10210 + }, + { + "epoch": 0.67, + "learning_rate": 1.4938628373666236e-06, + "logits/chosen": -2.25669264793396, + "logits/rejected": -2.3107972145080566, + "logps/chosen": -190.91799926757812, + "logps/rejected": -204.0850372314453, + "loss": 2016.073, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.18338727951049805, + "rewards/margins": 0.11294351518154144, + "rewards/rejected": -0.29633083939552307, + "step": 10220 + }, + { + "epoch": 0.67, + "learning_rate": 1.4886383868404203e-06, + "logits/chosen": -2.1663084030151367, + "logits/rejected": -2.0342824459075928, + "logps/chosen": -180.13150024414062, + "logps/rejected": -188.49063110351562, + "loss": 1373.7174, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18909737467765808, + "rewards/margins": 0.1838906705379486, + "rewards/rejected": -0.3729880750179291, + "step": 10230 + }, + { + "epoch": 0.67, + "learning_rate": 1.483419212071813e-06, + "logits/chosen": -2.12137770652771, + "logits/rejected": -1.9535928964614868, + "logps/chosen": -203.91366577148438, + "logps/rejected": -209.3536376953125, + "loss": 1905.7352, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15449532866477966, + "rewards/margins": 0.1262134462594986, + "rewards/rejected": -0.28070876002311707, + "step": 10240 + }, + { + "epoch": 0.67, + "learning_rate": 1.478205340286573e-06, + "logits/chosen": -2.257779121398926, + "logits/rejected": -2.225194215774536, + "logps/chosen": -225.85525512695312, + "logps/rejected": -220.8550567626953, + "loss": 1872.184, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.222463458776474, + "rewards/margins": 0.13004586100578308, + "rewards/rejected": -0.35250934958457947, + "step": 10250 + }, + { + "epoch": 0.67, + "learning_rate": 1.4729967986828104e-06, + "logits/chosen": -2.408285140991211, + "logits/rejected": -2.1716907024383545, + "logps/chosen": -335.02398681640625, + "logps/rejected": -307.013427734375, + "loss": 1414.1995, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1304025650024414, + "rewards/margins": 0.19957467913627625, + "rewards/rejected": -0.32997727394104004, + "step": 10260 + }, + { + "epoch": 0.67, + "learning_rate": 1.4677936144308286e-06, + "logits/chosen": -2.3844265937805176, + "logits/rejected": -2.093235731124878, + "logps/chosen": -239.69839477539062, + "logps/rejected": -229.95849609375, + "loss": 1554.3381, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15453621745109558, + "rewards/margins": 0.18323197960853577, + "rewards/rejected": -0.33776822686195374, + "step": 10270 + }, + { + "epoch": 0.67, + "learning_rate": 1.4625958146729864e-06, + "logits/chosen": -2.401050329208374, + "logits/rejected": -2.231168031692505, + "logps/chosen": -234.28488159179688, + "logps/rejected": -232.4915771484375, + "loss": 1820.0857, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14021024107933044, + "rewards/margins": 0.14154712855815887, + "rewards/rejected": -0.2817573845386505, + "step": 10280 + }, + { + "epoch": 0.67, + "learning_rate": 1.4574034265235523e-06, + "logits/chosen": -2.496002197265625, + "logits/rejected": -1.9339748620986938, + "logps/chosen": -269.0251770019531, + "logps/rejected": -190.8519287109375, + "loss": 1758.001, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1319662183523178, + "rewards/margins": 0.1958966851234436, + "rewards/rejected": -0.3278628885746002, + "step": 10290 + }, + { + "epoch": 0.67, + "learning_rate": 1.452216477068568e-06, + "logits/chosen": -2.3668737411499023, + "logits/rejected": -1.8553102016448975, + "logps/chosen": -242.12954711914062, + "logps/rejected": -170.4160614013672, + "loss": 1441.0404, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1208452582359314, + "rewards/margins": 0.21201328933238983, + "rewards/rejected": -0.3328585624694824, + "step": 10300 + }, + { + "epoch": 0.67, + "eval_logits/chosen": -2.336395502090454, + "eval_logits/rejected": -2.1501612663269043, + "eval_logps/chosen": -247.1606903076172, + "eval_logps/rejected": -242.07144165039062, + "eval_loss": 1778.0972900390625, + "eval_rewards/accuracies": 0.6549999713897705, + "eval_rewards/chosen": -0.1515573114156723, + "eval_rewards/margins": 0.1530378758907318, + "eval_rewards/rejected": -0.30459514260292053, + "eval_runtime": 726.5012, + "eval_samples_per_second": 2.753, + "eval_steps_per_second": 1.376, + "step": 10300 + }, + { + "epoch": 0.67, + "learning_rate": 1.4470349933657004e-06, + "logits/chosen": -2.5289347171783447, + "logits/rejected": -2.299401044845581, + "logps/chosen": -235.0850830078125, + "logps/rejected": -227.1444091796875, + "loss": 1535.2778, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14355462789535522, + "rewards/margins": 0.16038207709789276, + "rewards/rejected": -0.3039367198944092, + "step": 10310 + }, + { + "epoch": 0.68, + "learning_rate": 1.4418590024441096e-06, + "logits/chosen": -2.4282679557800293, + "logits/rejected": -2.0404889583587646, + "logps/chosen": -264.26629638671875, + "logps/rejected": -212.7019500732422, + "loss": 1732.6947, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10806041955947876, + "rewards/margins": 0.14920981228351593, + "rewards/rejected": -0.25727027654647827, + "step": 10320 + }, + { + "epoch": 0.68, + "learning_rate": 1.436688531304297e-06, + "logits/chosen": -2.422285556793213, + "logits/rejected": -2.0882749557495117, + "logps/chosen": -230.7120361328125, + "logps/rejected": -241.85488891601562, + "loss": 1907.8205, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1238451600074768, + "rewards/margins": 0.16632047295570374, + "rewards/rejected": -0.29016566276550293, + "step": 10330 + }, + { + "epoch": 0.68, + "learning_rate": 1.431523606917974e-06, + "logits/chosen": -2.2472996711730957, + "logits/rejected": -2.2454934120178223, + "logps/chosen": -224.62149047851562, + "logps/rejected": -243.8383026123047, + "loss": 2174.4451, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.19522175192832947, + "rewards/margins": 0.12602083384990692, + "rewards/rejected": -0.3212426006793976, + "step": 10340 + }, + { + "epoch": 0.68, + "learning_rate": 1.4263642562279162e-06, + "logits/chosen": -2.0353946685791016, + "logits/rejected": -2.0154049396514893, + "logps/chosen": -265.7103271484375, + "logps/rejected": -288.4093322753906, + "loss": 1817.1635, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16222988069057465, + "rewards/margins": 0.14208927750587463, + "rewards/rejected": -0.3043191730976105, + "step": 10350 + }, + { + "epoch": 0.68, + "learning_rate": 1.4212105061478257e-06, + "logits/chosen": -2.087690830230713, + "logits/rejected": -2.112273693084717, + "logps/chosen": -246.8751220703125, + "logps/rejected": -272.3307189941406, + "loss": 1790.5391, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16984596848487854, + "rewards/margins": 0.1763208955526352, + "rewards/rejected": -0.34616684913635254, + "step": 10360 + }, + { + "epoch": 0.68, + "learning_rate": 1.4160623835621848e-06, + "logits/chosen": -2.43872332572937, + "logits/rejected": -2.3132472038269043, + "logps/chosen": -249.39242553710938, + "logps/rejected": -254.17178344726562, + "loss": 1500.7467, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1208123192191124, + "rewards/margins": 0.16257521510124207, + "rewards/rejected": -0.28338757157325745, + "step": 10370 + }, + { + "epoch": 0.68, + "learning_rate": 1.4109199153261249e-06, + "logits/chosen": -2.226412773132324, + "logits/rejected": -2.1237289905548096, + "logps/chosen": -289.9986267089844, + "logps/rejected": -275.4288024902344, + "loss": 1812.118, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14004027843475342, + "rewards/margins": 0.1574375331401825, + "rewards/rejected": -0.2974777817726135, + "step": 10380 + }, + { + "epoch": 0.68, + "learning_rate": 1.405783128265278e-06, + "logits/chosen": -2.304079294204712, + "logits/rejected": -2.284876823425293, + "logps/chosen": -222.9115753173828, + "logps/rejected": -230.6747589111328, + "loss": 1979.2328, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1877807080745697, + "rewards/margins": 0.11029338836669922, + "rewards/rejected": -0.2980740964412689, + "step": 10390 + }, + { + "epoch": 0.68, + "learning_rate": 1.4006520491756427e-06, + "logits/chosen": -2.4129700660705566, + "logits/rejected": -2.1868672370910645, + "logps/chosen": -209.59365844726562, + "logps/rejected": -160.5478973388672, + "loss": 1606.4429, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13369429111480713, + "rewards/margins": 0.15595155954360962, + "rewards/rejected": -0.28964588046073914, + "step": 10400 + }, + { + "epoch": 0.68, + "eval_logits/chosen": -2.336988925933838, + "eval_logits/rejected": -2.1507012844085693, + "eval_logps/chosen": -243.77908325195312, + "eval_logps/rejected": -238.7640380859375, + "eval_loss": 1776.54150390625, + "eval_rewards/accuracies": 0.6604999899864197, + "eval_rewards/chosen": -0.11774113774299622, + "eval_rewards/margins": 0.15378008782863617, + "eval_rewards/rejected": -0.2715212106704712, + "eval_runtime": 723.8472, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.382, + "step": 10400 + }, + { + "epoch": 0.68, + "learning_rate": 1.39552670482344e-06, + "logits/chosen": -2.234830856323242, + "logits/rejected": -2.3060553073883057, + "logps/chosen": -187.51327514648438, + "logps/rejected": -200.25709533691406, + "loss": 1896.8854, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12913690507411957, + "rewards/margins": 0.1285235583782196, + "rewards/rejected": -0.25766047835350037, + "step": 10410 + }, + { + "epoch": 0.68, + "learning_rate": 1.3904071219449776e-06, + "logits/chosen": -2.319739818572998, + "logits/rejected": -1.888177514076233, + "logps/chosen": -208.35977172851562, + "logps/rejected": -139.51470947265625, + "loss": 1721.7918, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0845813974738121, + "rewards/margins": 0.15815046429634094, + "rewards/rejected": -0.24273185431957245, + "step": 10420 + }, + { + "epoch": 0.68, + "learning_rate": 1.3852933272465068e-06, + "logits/chosen": -2.4715352058410645, + "logits/rejected": -2.241602659225464, + "logps/chosen": -245.01632690429688, + "logps/rejected": -224.55087280273438, + "loss": 1806.8795, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.07156185060739517, + "rewards/margins": 0.14758458733558655, + "rewards/rejected": -0.21914644539356232, + "step": 10430 + }, + { + "epoch": 0.68, + "learning_rate": 1.3801853474040873e-06, + "logits/chosen": -2.273109197616577, + "logits/rejected": -2.209038734436035, + "logps/chosen": -250.9441375732422, + "logps/rejected": -252.0476837158203, + "loss": 1534.3904, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12422917783260345, + "rewards/margins": 0.17563822865486145, + "rewards/rejected": -0.2998674213886261, + "step": 10440 + }, + { + "epoch": 0.68, + "learning_rate": 1.3750832090634417e-06, + "logits/chosen": -2.4183297157287598, + "logits/rejected": -2.111826181411743, + "logps/chosen": -193.4394073486328, + "logps/rejected": -192.63906860351562, + "loss": 1852.5627, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.07266728579998016, + "rewards/margins": 0.1385105401277542, + "rewards/rejected": -0.211177796125412, + "step": 10450 + }, + { + "epoch": 0.68, + "learning_rate": 1.3699869388398245e-06, + "logits/chosen": -2.278883218765259, + "logits/rejected": -2.1369309425354004, + "logps/chosen": -225.41000366210938, + "logps/rejected": -223.36270141601562, + "loss": 1772.0594, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12388720363378525, + "rewards/margins": 0.1640610694885254, + "rewards/rejected": -0.28794828057289124, + "step": 10460 + }, + { + "epoch": 0.69, + "learning_rate": 1.3648965633178772e-06, + "logits/chosen": -2.3005356788635254, + "logits/rejected": -2.2080466747283936, + "logps/chosen": -210.2812042236328, + "logps/rejected": -241.01046752929688, + "loss": 1483.6773, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09549007564783096, + "rewards/margins": 0.19317705929279327, + "rewards/rejected": -0.28866714239120483, + "step": 10470 + }, + { + "epoch": 0.69, + "learning_rate": 1.3598121090514938e-06, + "logits/chosen": -2.290022850036621, + "logits/rejected": -2.1660006046295166, + "logps/chosen": -193.58340454101562, + "logps/rejected": -185.4873809814453, + "loss": 1735.5125, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10575524717569351, + "rewards/margins": 0.16027264297008514, + "rewards/rejected": -0.26602789759635925, + "step": 10480 + }, + { + "epoch": 0.69, + "learning_rate": 1.3547336025636753e-06, + "logits/chosen": -2.218031167984009, + "logits/rejected": -2.0019259452819824, + "logps/chosen": -283.0633544921875, + "logps/rejected": -257.22479248046875, + "loss": 1576.8256, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07766801863908768, + "rewards/margins": 0.1569044291973114, + "rewards/rejected": -0.23457245528697968, + "step": 10490 + }, + { + "epoch": 0.69, + "learning_rate": 1.3496610703464022e-06, + "logits/chosen": -2.36081600189209, + "logits/rejected": -2.132767915725708, + "logps/chosen": -239.56436157226562, + "logps/rejected": -205.96337890625, + "loss": 2053.1404, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10349766165018082, + "rewards/margins": 0.11920388042926788, + "rewards/rejected": -0.2227015495300293, + "step": 10500 + }, + { + "epoch": 0.69, + "eval_logits/chosen": -2.3365676403045654, + "eval_logits/rejected": -2.1502785682678223, + "eval_logps/chosen": -241.38241577148438, + "eval_logps/rejected": -236.80250549316406, + "eval_loss": 1771.5421142578125, + "eval_rewards/accuracies": 0.6620000004768372, + "eval_rewards/chosen": -0.09377462416887283, + "eval_rewards/margins": 0.15813127160072327, + "eval_rewards/rejected": -0.2519059181213379, + "eval_runtime": 728.0246, + "eval_samples_per_second": 2.747, + "eval_steps_per_second": 1.374, + "step": 10500 + }, + { + "epoch": 0.69, + "learning_rate": 1.3445945388604848e-06, + "logits/chosen": -2.2319271564483643, + "logits/rejected": -1.9931703805923462, + "logps/chosen": -253.85574340820312, + "logps/rejected": -228.87545776367188, + "loss": 1665.5861, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.15632253885269165, + "rewards/margins": 0.18180391192436218, + "rewards/rejected": -0.33812645077705383, + "step": 10510 + }, + { + "epoch": 0.69, + "learning_rate": 1.3395340345354358e-06, + "logits/chosen": -2.257664203643799, + "logits/rejected": -2.386795997619629, + "logps/chosen": -237.11557006835938, + "logps/rejected": -264.43096923828125, + "loss": 2022.6344, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13417726755142212, + "rewards/margins": 0.1006779819726944, + "rewards/rejected": -0.2348552644252777, + "step": 10520 + }, + { + "epoch": 0.69, + "learning_rate": 1.334479583769322e-06, + "logits/chosen": -2.4861791133880615, + "logits/rejected": -2.179697036743164, + "logps/chosen": -262.93231201171875, + "logps/rejected": -225.8566131591797, + "loss": 1849.49, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.08093537390232086, + "rewards/margins": 0.128939688205719, + "rewards/rejected": -0.20987506210803986, + "step": 10530 + }, + { + "epoch": 0.69, + "learning_rate": 1.3294312129286366e-06, + "logits/chosen": -2.3211584091186523, + "logits/rejected": -2.2067387104034424, + "logps/chosen": -271.5013732910156, + "logps/rejected": -275.35357666015625, + "loss": 1424.4191, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.0477275513112545, + "rewards/margins": 0.17523090541362762, + "rewards/rejected": -0.22295847535133362, + "step": 10540 + }, + { + "epoch": 0.69, + "learning_rate": 1.324388948348153e-06, + "logits/chosen": -2.482022762298584, + "logits/rejected": -2.0950958728790283, + "logps/chosen": -292.1917419433594, + "logps/rejected": -226.5915985107422, + "loss": 1598.9847, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.06953771412372589, + "rewards/margins": 0.15766309201717377, + "rewards/rejected": -0.22720082104206085, + "step": 10550 + }, + { + "epoch": 0.69, + "learning_rate": 1.319352816330796e-06, + "logits/chosen": -2.5680453777313232, + "logits/rejected": -2.056453227996826, + "logps/chosen": -293.2705383300781, + "logps/rejected": -205.46707153320312, + "loss": 1670.0934, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10073323547840118, + "rewards/margins": 0.1865191012620926, + "rewards/rejected": -0.2872523367404938, + "step": 10560 + }, + { + "epoch": 0.69, + "learning_rate": 1.314322843147494e-06, + "logits/chosen": -2.173163652420044, + "logits/rejected": -2.2709197998046875, + "logps/chosen": -191.83360290527344, + "logps/rejected": -255.783447265625, + "loss": 2125.8488, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.21507766842842102, + "rewards/margins": 0.0908210277557373, + "rewards/rejected": -0.3058987259864807, + "step": 10570 + }, + { + "epoch": 0.69, + "learning_rate": 1.3092990550370526e-06, + "logits/chosen": -2.4326562881469727, + "logits/rejected": -2.095061779022217, + "logps/chosen": -358.76983642578125, + "logps/rejected": -290.0596008300781, + "loss": 1846.373, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13011455535888672, + "rewards/margins": 0.16640953719615936, + "rewards/rejected": -0.2965241074562073, + "step": 10580 + }, + { + "epoch": 0.69, + "learning_rate": 1.3042814782060131e-06, + "logits/chosen": -2.3822150230407715, + "logits/rejected": -2.0831844806671143, + "logps/chosen": -187.81179809570312, + "logps/rejected": -182.1324462890625, + "loss": 1811.2422, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.06308932602405548, + "rewards/margins": 0.1823693811893463, + "rewards/rejected": -0.245458722114563, + "step": 10590 + }, + { + "epoch": 0.69, + "learning_rate": 1.2992701388285112e-06, + "logits/chosen": -2.388526439666748, + "logits/rejected": -2.132236957550049, + "logps/chosen": -276.08428955078125, + "logps/rejected": -247.83102416992188, + "loss": 1666.0459, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.06735239177942276, + "rewards/margins": 0.16218677163124084, + "rewards/rejected": -0.229539155960083, + "step": 10600 + }, + { + "epoch": 0.69, + "eval_logits/chosen": -2.3377394676208496, + "eval_logits/rejected": -2.1511542797088623, + "eval_logps/chosen": -242.1644287109375, + "eval_logps/rejected": -237.53196716308594, + "eval_loss": 1766.10595703125, + "eval_rewards/accuracies": 0.6614999771118164, + "eval_rewards/chosen": -0.10159474611282349, + "eval_rewards/margins": 0.15760593116283417, + "eval_rewards/rejected": -0.25920066237449646, + "eval_runtime": 723.0366, + "eval_samples_per_second": 2.766, + "eval_steps_per_second": 1.383, + "step": 10600 + }, + { + "epoch": 0.69, + "learning_rate": 1.29426506304615e-06, + "logits/chosen": -2.2357826232910156, + "logits/rejected": -2.1712448596954346, + "logps/chosen": -238.3498992919922, + "logps/rejected": -226.3423309326172, + "loss": 2526.1244, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.15626461803913116, + "rewards/margins": 0.0747048556804657, + "rewards/rejected": -0.23096947371959686, + "step": 10610 + }, + { + "epoch": 0.69, + "learning_rate": 1.289266276967855e-06, + "logits/chosen": -2.375516653060913, + "logits/rejected": -2.2651820182800293, + "logps/chosen": -345.6637878417969, + "logps/rejected": -274.7169189453125, + "loss": 1486.8291, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07081923633813858, + "rewards/margins": 0.1921137273311615, + "rewards/rejected": -0.2629329562187195, + "step": 10620 + }, + { + "epoch": 0.7, + "learning_rate": 1.284273806669745e-06, + "logits/chosen": -2.3371615409851074, + "logits/rejected": -2.1077303886413574, + "logps/chosen": -256.08428955078125, + "logps/rejected": -286.40386962890625, + "loss": 1893.0041, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.16759343445301056, + "rewards/margins": 0.12778960168361664, + "rewards/rejected": -0.2953830361366272, + "step": 10630 + }, + { + "epoch": 0.7, + "learning_rate": 1.2792876781949884e-06, + "logits/chosen": -2.083141326904297, + "logits/rejected": -1.8333759307861328, + "logps/chosen": -220.6055450439453, + "logps/rejected": -211.8660430908203, + "loss": 2060.8947, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.14532899856567383, + "rewards/margins": 0.10578174889087677, + "rewards/rejected": -0.2511107325553894, + "step": 10640 + }, + { + "epoch": 0.7, + "learning_rate": 1.274307917553676e-06, + "logits/chosen": -2.3342247009277344, + "logits/rejected": -2.2580673694610596, + "logps/chosen": -208.902587890625, + "logps/rejected": -252.3454132080078, + "loss": 1515.762, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13243810832500458, + "rewards/margins": 0.18843302130699158, + "rewards/rejected": -0.32087117433547974, + "step": 10650 + }, + { + "epoch": 0.7, + "learning_rate": 1.2693345507226767e-06, + "logits/chosen": -2.127349376678467, + "logits/rejected": -2.2174124717712402, + "logps/chosen": -243.489990234375, + "logps/rejected": -262.34393310546875, + "loss": 1637.8714, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13499729335308075, + "rewards/margins": 0.1674044132232666, + "rewards/rejected": -0.30240169167518616, + "step": 10660 + }, + { + "epoch": 0.7, + "learning_rate": 1.2643676036455099e-06, + "logits/chosen": -2.3676364421844482, + "logits/rejected": -2.281890869140625, + "logps/chosen": -299.4869079589844, + "logps/rejected": -263.0523986816406, + "loss": 1848.6758, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11634413152933121, + "rewards/margins": 0.1430656611919403, + "rewards/rejected": -0.2594097852706909, + "step": 10670 + }, + { + "epoch": 0.7, + "learning_rate": 1.259407102232203e-06, + "logits/chosen": -2.471794366836548, + "logits/rejected": -2.0974783897399902, + "logps/chosen": -290.4842224121094, + "logps/rejected": -236.22451782226562, + "loss": 1389.6499, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1284985989332199, + "rewards/margins": 0.18684282898902893, + "rewards/rejected": -0.3153414726257324, + "step": 10680 + }, + { + "epoch": 0.7, + "learning_rate": 1.254453072359163e-06, + "logits/chosen": -2.410886287689209, + "logits/rejected": -2.183062791824341, + "logps/chosen": -247.2122039794922, + "logps/rejected": -240.69680786132812, + "loss": 1719.7668, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13192158937454224, + "rewards/margins": 0.1535084843635559, + "rewards/rejected": -0.28543007373809814, + "step": 10690 + }, + { + "epoch": 0.7, + "learning_rate": 1.2495055398690337e-06, + "logits/chosen": -2.4857728481292725, + "logits/rejected": -2.243335723876953, + "logps/chosen": -241.6734161376953, + "logps/rejected": -244.768798828125, + "loss": 2062.3629, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.11819012463092804, + "rewards/margins": 0.12378589808940887, + "rewards/rejected": -0.24197602272033691, + "step": 10700 + }, + { + "epoch": 0.7, + "eval_logits/chosen": -2.340343475341797, + "eval_logits/rejected": -2.1535558700561523, + "eval_logps/chosen": -244.5763702392578, + "eval_logps/rejected": -240.011962890625, + "eval_loss": 1763.6278076171875, + "eval_rewards/accuracies": 0.6635000109672546, + "eval_rewards/chosen": -0.12571415305137634, + "eval_rewards/margins": 0.15828652679920197, + "eval_rewards/rejected": -0.2840006947517395, + "eval_runtime": 722.9166, + "eval_samples_per_second": 2.767, + "eval_steps_per_second": 1.383, + "step": 10700 + }, + { + "epoch": 0.7, + "learning_rate": 1.2445645305705718e-06, + "logits/chosen": -2.511423110961914, + "logits/rejected": -2.175668239593506, + "logps/chosen": -220.97122192382812, + "logps/rejected": -212.86807250976562, + "loss": 2147.9316, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1465025693178177, + "rewards/margins": 0.12539730966091156, + "rewards/rejected": -0.27189984917640686, + "step": 10710 + }, + { + "epoch": 0.7, + "learning_rate": 1.2396300702384995e-06, + "logits/chosen": -2.497962474822998, + "logits/rejected": -2.2529397010803223, + "logps/chosen": -274.5304870605469, + "logps/rejected": -246.68466186523438, + "loss": 2249.2039, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.16722413897514343, + "rewards/margins": 0.09819474071264267, + "rewards/rejected": -0.2654188871383667, + "step": 10720 + }, + { + "epoch": 0.7, + "learning_rate": 1.234702184613381e-06, + "logits/chosen": -2.1192617416381836, + "logits/rejected": -2.2030041217803955, + "logps/chosen": -228.84054565429688, + "logps/rejected": -241.6710968017578, + "loss": 1799.9865, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1335824429988861, + "rewards/margins": 0.13166043162345886, + "rewards/rejected": -0.26524287462234497, + "step": 10730 + }, + { + "epoch": 0.7, + "learning_rate": 1.2297808994014793e-06, + "logits/chosen": -2.4309096336364746, + "logits/rejected": -2.181896209716797, + "logps/chosen": -296.35955810546875, + "logps/rejected": -274.7001647949219, + "loss": 1733.6844, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10424685478210449, + "rewards/margins": 0.1323317587375641, + "rewards/rejected": -0.23657865822315216, + "step": 10740 + }, + { + "epoch": 0.7, + "learning_rate": 1.2248662402746314e-06, + "logits/chosen": -2.2782938480377197, + "logits/rejected": -2.1423420906066895, + "logps/chosen": -205.86904907226562, + "logps/rejected": -215.0177001953125, + "loss": 1634.1285, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.15952368080615997, + "rewards/margins": 0.15264497697353363, + "rewards/rejected": -0.3121686577796936, + "step": 10750 + }, + { + "epoch": 0.7, + "learning_rate": 1.2199582328701045e-06, + "logits/chosen": -2.4306654930114746, + "logits/rejected": -1.937919020652771, + "logps/chosen": -302.0787658691406, + "logps/rejected": -277.5871887207031, + "loss": 1759.1566, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11445232480764389, + "rewards/margins": 0.15799331665039062, + "rewards/rejected": -0.2724456489086151, + "step": 10760 + }, + { + "epoch": 0.7, + "learning_rate": 1.2150569027904712e-06, + "logits/chosen": -2.3267223834991455, + "logits/rejected": -2.2170944213867188, + "logps/chosen": -257.29571533203125, + "logps/rejected": -268.56219482421875, + "loss": 1791.2227, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1131223812699318, + "rewards/margins": 0.17569416761398315, + "rewards/rejected": -0.28881654143333435, + "step": 10770 + }, + { + "epoch": 0.71, + "learning_rate": 1.2101622756034688e-06, + "logits/chosen": -2.318678379058838, + "logits/rejected": -2.3083908557891846, + "logps/chosen": -230.8387908935547, + "logps/rejected": -215.8868408203125, + "loss": 1718.6527, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.0808473750948906, + "rewards/margins": 0.15413309633731842, + "rewards/rejected": -0.2349804937839508, + "step": 10780 + }, + { + "epoch": 0.71, + "learning_rate": 1.2052743768418715e-06, + "logits/chosen": -2.3797993659973145, + "logits/rejected": -2.1624557971954346, + "logps/chosen": -256.75262451171875, + "logps/rejected": -237.8788604736328, + "loss": 1739.4143, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08943568170070648, + "rewards/margins": 0.1513310670852661, + "rewards/rejected": -0.2407667636871338, + "step": 10790 + }, + { + "epoch": 0.71, + "learning_rate": 1.2003932320033523e-06, + "logits/chosen": -2.4958038330078125, + "logits/rejected": -2.1804280281066895, + "logps/chosen": -237.180908203125, + "logps/rejected": -261.7378234863281, + "loss": 1241.0871, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10678191483020782, + "rewards/margins": 0.2392505705356598, + "rewards/rejected": -0.3460325300693512, + "step": 10800 + }, + { + "epoch": 0.71, + "eval_logits/chosen": -2.347846031188965, + "eval_logits/rejected": -2.160001039505005, + "eval_logps/chosen": -243.9580841064453, + "eval_logps/rejected": -239.3552703857422, + "eval_loss": 1766.05029296875, + "eval_rewards/accuracies": 0.6635000109672546, + "eval_rewards/chosen": -0.11953125149011612, + "eval_rewards/margins": 0.1579025387763977, + "eval_rewards/rejected": -0.2774338126182556, + "eval_runtime": 724.7871, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 10800 + }, + { + "epoch": 0.71, + "learning_rate": 1.1955188665503553e-06, + "logits/chosen": -2.1565940380096436, + "logits/rejected": -2.215562105178833, + "logps/chosen": -219.93881225585938, + "logps/rejected": -212.1143035888672, + "loss": 2009.3404, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.14815133810043335, + "rewards/margins": 0.12965616583824158, + "rewards/rejected": -0.2778075337409973, + "step": 10810 + }, + { + "epoch": 0.71, + "learning_rate": 1.1906513059099566e-06, + "logits/chosen": -2.3357901573181152, + "logits/rejected": -2.0216660499572754, + "logps/chosen": -244.9086456298828, + "logps/rejected": -254.49697875976562, + "loss": 1451.7775, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.13419561088085175, + "rewards/margins": 0.1832888424396515, + "rewards/rejected": -0.31748443841934204, + "step": 10820 + }, + { + "epoch": 0.71, + "learning_rate": 1.185790575473738e-06, + "logits/chosen": -2.277193069458008, + "logits/rejected": -2.0780417919158936, + "logps/chosen": -241.35208129882812, + "logps/rejected": -210.4105987548828, + "loss": 1686.5461, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12218759208917618, + "rewards/margins": 0.1633271425962448, + "rewards/rejected": -0.2855147123336792, + "step": 10830 + }, + { + "epoch": 0.71, + "learning_rate": 1.1809367005976516e-06, + "logits/chosen": -2.3439080715179443, + "logits/rejected": -2.101081609725952, + "logps/chosen": -293.42987060546875, + "logps/rejected": -234.614501953125, + "loss": 1877.9219, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.10935065895318985, + "rewards/margins": 0.12232010066509247, + "rewards/rejected": -0.23167076706886292, + "step": 10840 + }, + { + "epoch": 0.71, + "learning_rate": 1.1760897066018842e-06, + "logits/chosen": -2.271735668182373, + "logits/rejected": -2.1386935710906982, + "logps/chosen": -234.00830078125, + "logps/rejected": -246.95761108398438, + "loss": 1651.3365, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.109672412276268, + "rewards/margins": 0.18946237862110138, + "rewards/rejected": -0.2991347908973694, + "step": 10850 + }, + { + "epoch": 0.71, + "learning_rate": 1.1712496187707327e-06, + "logits/chosen": -2.307668447494507, + "logits/rejected": -2.053638219833374, + "logps/chosen": -254.58633422851562, + "logps/rejected": -263.89300537109375, + "loss": 1443.3568, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.14282092452049255, + "rewards/margins": 0.2337246686220169, + "rewards/rejected": -0.37654557824134827, + "step": 10860 + }, + { + "epoch": 0.71, + "learning_rate": 1.1664164623524646e-06, + "logits/chosen": -2.286165237426758, + "logits/rejected": -2.126422882080078, + "logps/chosen": -228.3876953125, + "logps/rejected": -217.5677490234375, + "loss": 1624.3861, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10012678802013397, + "rewards/margins": 0.17437371611595154, + "rewards/rejected": -0.2745005190372467, + "step": 10870 + }, + { + "epoch": 0.71, + "learning_rate": 1.1615902625591926e-06, + "logits/chosen": -2.244960308074951, + "logits/rejected": -2.16121244430542, + "logps/chosen": -243.59188842773438, + "logps/rejected": -248.5153350830078, + "loss": 1472.7349, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1837068498134613, + "rewards/margins": 0.14337095618247986, + "rewards/rejected": -0.32707780599594116, + "step": 10880 + }, + { + "epoch": 0.71, + "learning_rate": 1.156771044566738e-06, + "logits/chosen": -2.3289358615875244, + "logits/rejected": -2.2567138671875, + "logps/chosen": -279.33380126953125, + "logps/rejected": -250.97756958007812, + "loss": 1849.6008, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.14719167351722717, + "rewards/margins": 0.14861159026622772, + "rewards/rejected": -0.2958032786846161, + "step": 10890 + }, + { + "epoch": 0.71, + "learning_rate": 1.1519588335145037e-06, + "logits/chosen": -2.284789800643921, + "logits/rejected": -2.4238369464874268, + "logps/chosen": -221.6291961669922, + "logps/rejected": -253.0560302734375, + "loss": 1870.9098, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14050407707691193, + "rewards/margins": 0.12727408111095428, + "rewards/rejected": -0.2677781581878662, + "step": 10900 + }, + { + "epoch": 0.71, + "eval_logits/chosen": -2.345259189605713, + "eval_logits/rejected": -2.1578006744384766, + "eval_logps/chosen": -244.84068298339844, + "eval_logps/rejected": -240.75035095214844, + "eval_loss": 1764.2947998046875, + "eval_rewards/accuracies": 0.6620000004768372, + "eval_rewards/chosen": -0.12835733592510223, + "eval_rewards/margins": 0.16302713751792908, + "eval_rewards/rejected": -0.29138442873954773, + "eval_runtime": 723.9578, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.381, + "step": 10900 + }, + { + "epoch": 0.71, + "learning_rate": 1.1471536545053382e-06, + "logits/chosen": -2.3750205039978027, + "logits/rejected": -2.3602657318115234, + "logps/chosen": -217.36862182617188, + "logps/rejected": -239.7703857421875, + "loss": 1909.0926, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08333705365657806, + "rewards/margins": 0.14849582314491272, + "rewards/rejected": -0.23183289170265198, + "step": 10910 + }, + { + "epoch": 0.71, + "learning_rate": 1.1423555326054112e-06, + "logits/chosen": -2.281400442123413, + "logits/rejected": -2.011650562286377, + "logps/chosen": -295.34649658203125, + "logps/rejected": -246.96923828125, + "loss": 1257.3419, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.12511949241161346, + "rewards/margins": 0.2505880296230316, + "rewards/rejected": -0.3757075369358063, + "step": 10920 + }, + { + "epoch": 0.72, + "learning_rate": 1.1375644928440743e-06, + "logits/chosen": -2.4245126247406006, + "logits/rejected": -1.997741937637329, + "logps/chosen": -246.53213500976562, + "logps/rejected": -196.3480987548828, + "loss": 1607.6604, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.0984887033700943, + "rewards/margins": 0.17319846153259277, + "rewards/rejected": -0.2716871201992035, + "step": 10930 + }, + { + "epoch": 0.72, + "learning_rate": 1.1327805602137396e-06, + "logits/chosen": -2.355712413787842, + "logits/rejected": -2.1805901527404785, + "logps/chosen": -270.1285400390625, + "logps/rejected": -235.70108032226562, + "loss": 1566.6525, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.11384463310241699, + "rewards/margins": 0.18550507724285126, + "rewards/rejected": -0.29934969544410706, + "step": 10940 + }, + { + "epoch": 0.72, + "learning_rate": 1.1280037596697426e-06, + "logits/chosen": -2.211615562438965, + "logits/rejected": -2.1793317794799805, + "logps/chosen": -237.2572784423828, + "logps/rejected": -293.44342041015625, + "loss": 1663.5357, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.16374829411506653, + "rewards/margins": 0.1925935000181198, + "rewards/rejected": -0.35634177923202515, + "step": 10950 + }, + { + "epoch": 0.72, + "learning_rate": 1.123234116130216e-06, + "logits/chosen": -2.2696049213409424, + "logits/rejected": -2.2383406162261963, + "logps/chosen": -202.56617736816406, + "logps/rejected": -223.7290802001953, + "loss": 2012.6854, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16677790880203247, + "rewards/margins": 0.15108618140220642, + "rewards/rejected": -0.3178640902042389, + "step": 10960 + }, + { + "epoch": 0.72, + "learning_rate": 1.1184716544759553e-06, + "logits/chosen": -2.1331257820129395, + "logits/rejected": -2.110107183456421, + "logps/chosen": -176.7662811279297, + "logps/rejected": -200.56005859375, + "loss": 1832.0801, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11402100324630737, + "rewards/margins": 0.11379198729991913, + "rewards/rejected": -0.2278130054473877, + "step": 10970 + }, + { + "epoch": 0.72, + "learning_rate": 1.1137163995502948e-06, + "logits/chosen": -2.558546781539917, + "logits/rejected": -2.3269588947296143, + "logps/chosen": -231.333740234375, + "logps/rejected": -213.76669311523438, + "loss": 1781.4242, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09798694401979446, + "rewards/margins": 0.15134866535663605, + "rewards/rejected": -0.24933557212352753, + "step": 10980 + }, + { + "epoch": 0.72, + "learning_rate": 1.1089683761589717e-06, + "logits/chosen": -2.1864194869995117, + "logits/rejected": -2.0251612663269043, + "logps/chosen": -249.9845733642578, + "logps/rejected": -255.09475708007812, + "loss": 1544.3908, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09778235107660294, + "rewards/margins": 0.2402438372373581, + "rewards/rejected": -0.338026225566864, + "step": 10990 + }, + { + "epoch": 0.72, + "learning_rate": 1.1042276090700044e-06, + "logits/chosen": -2.3458712100982666, + "logits/rejected": -2.284625291824341, + "logps/chosen": -229.9285125732422, + "logps/rejected": -269.467529296875, + "loss": 2322.8574, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.1962636113166809, + "rewards/margins": 0.0936969667673111, + "rewards/rejected": -0.2899605631828308, + "step": 11000 + }, + { + "epoch": 0.72, + "eval_logits/chosen": -2.339794874191284, + "eval_logits/rejected": -2.152827262878418, + "eval_logps/chosen": -245.14317321777344, + "eval_logps/rejected": -240.09378051757812, + "eval_loss": 1763.4559326171875, + "eval_rewards/accuracies": 0.6639999747276306, + "eval_rewards/chosen": -0.1313822716474533, + "eval_rewards/margins": 0.15343663096427917, + "eval_rewards/rejected": -0.2848189175128937, + "eval_runtime": 724.1879, + "eval_samples_per_second": 2.762, + "eval_steps_per_second": 1.381, + "step": 11000 + }, + { + "epoch": 0.72, + "learning_rate": 1.0994941230135536e-06, + "logits/chosen": -2.305234909057617, + "logits/rejected": -2.010282516479492, + "logps/chosen": -241.6505889892578, + "logps/rejected": -231.662841796875, + "loss": 1076.5374, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08800555020570755, + "rewards/margins": 0.26072031259536743, + "rewards/rejected": -0.3487258553504944, + "step": 11010 + }, + { + "epoch": 0.72, + "learning_rate": 1.094767942681804e-06, + "logits/chosen": -2.582350254058838, + "logits/rejected": -2.1599652767181396, + "logps/chosen": -269.214599609375, + "logps/rejected": -243.85092163085938, + "loss": 1959.1641, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2134559601545334, + "rewards/margins": 0.1383691430091858, + "rewards/rejected": -0.35182514786720276, + "step": 11020 + }, + { + "epoch": 0.72, + "learning_rate": 1.0900490927288248e-06, + "logits/chosen": -2.12015700340271, + "logits/rejected": -2.0761804580688477, + "logps/chosen": -274.0080261230469, + "logps/rejected": -238.61367797851562, + "loss": 1776.8357, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13539735972881317, + "rewards/margins": 0.1408197581768036, + "rewards/rejected": -0.27621710300445557, + "step": 11030 + }, + { + "epoch": 0.72, + "learning_rate": 1.0853375977704511e-06, + "logits/chosen": -2.343122959136963, + "logits/rejected": -2.161822557449341, + "logps/chosen": -256.9599304199219, + "logps/rejected": -207.81494140625, + "loss": 1541.3028, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16535378992557526, + "rewards/margins": 0.14865735173225403, + "rewards/rejected": -0.3140111565589905, + "step": 11040 + }, + { + "epoch": 0.72, + "learning_rate": 1.0806334823841466e-06, + "logits/chosen": -2.193089246749878, + "logits/rejected": -2.2982065677642822, + "logps/chosen": -257.24639892578125, + "logps/rejected": -295.80499267578125, + "loss": 2358.9592, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.17630498111248016, + "rewards/margins": 0.09155434370040894, + "rewards/rejected": -0.2678592801094055, + "step": 11050 + }, + { + "epoch": 0.72, + "learning_rate": 1.0759367711088825e-06, + "logits/chosen": -2.201061487197876, + "logits/rejected": -2.335559368133545, + "logps/chosen": -199.49998474121094, + "logps/rejected": -242.69033813476562, + "loss": 1907.6775, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15667478740215302, + "rewards/margins": 0.09713147580623627, + "rewards/rejected": -0.25380629301071167, + "step": 11060 + }, + { + "epoch": 0.72, + "learning_rate": 1.0712474884450056e-06, + "logits/chosen": -2.3094496726989746, + "logits/rejected": -2.110459566116333, + "logps/chosen": -214.3277587890625, + "logps/rejected": -203.27700805664062, + "loss": 1435.9635, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11494525521993637, + "rewards/margins": 0.17736020684242249, + "rewards/rejected": -0.29230546951293945, + "step": 11070 + }, + { + "epoch": 0.72, + "learning_rate": 1.066565658854112e-06, + "logits/chosen": -2.2599921226501465, + "logits/rejected": -2.2243447303771973, + "logps/chosen": -142.46395874023438, + "logps/rejected": -153.2086181640625, + "loss": 1846.8561, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.14632046222686768, + "rewards/margins": 0.1397717297077179, + "rewards/rejected": -0.2860921621322632, + "step": 11080 + }, + { + "epoch": 0.73, + "learning_rate": 1.0618913067589165e-06, + "logits/chosen": -2.3570351600646973, + "logits/rejected": -2.141935348510742, + "logps/chosen": -228.3729248046875, + "logps/rejected": -207.08169555664062, + "loss": 1559.0029, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12558218836784363, + "rewards/margins": 0.16168659925460815, + "rewards/rejected": -0.2872687876224518, + "step": 11090 + }, + { + "epoch": 0.73, + "learning_rate": 1.0572244565431313e-06, + "logits/chosen": -2.2187297344207764, + "logits/rejected": -2.1026558876037598, + "logps/chosen": -161.37313842773438, + "logps/rejected": -179.53013610839844, + "loss": 1666.5447, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.15956351161003113, + "rewards/margins": 0.12995785474777222, + "rewards/rejected": -0.28952136635780334, + "step": 11100 + }, + { + "epoch": 0.73, + "eval_logits/chosen": -2.3374075889587402, + "eval_logits/rejected": -2.1504600048065186, + "eval_logps/chosen": -245.01600646972656, + "eval_logps/rejected": -240.04672241210938, + "eval_loss": 1763.682861328125, + "eval_rewards/accuracies": 0.6589999794960022, + "eval_rewards/chosen": -0.13011065125465393, + "eval_rewards/margins": 0.15423738956451416, + "eval_rewards/rejected": -0.2843480408191681, + "eval_runtime": 725.2445, + "eval_samples_per_second": 2.758, + "eval_steps_per_second": 1.379, + "step": 11100 + }, + { + "epoch": 0.73, + "learning_rate": 1.0525651325513317e-06, + "logits/chosen": -2.3347012996673584, + "logits/rejected": -2.3130810260772705, + "logps/chosen": -344.20343017578125, + "logps/rejected": -334.12054443359375, + "loss": 2193.3535, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.12933756411075592, + "rewards/margins": 0.11072547733783722, + "rewards/rejected": -0.24006302654743195, + "step": 11110 + }, + { + "epoch": 0.73, + "learning_rate": 1.0479133590888351e-06, + "logits/chosen": -2.378882884979248, + "logits/rejected": -2.0841665267944336, + "logps/chosen": -268.0919494628906, + "logps/rejected": -256.08404541015625, + "loss": 1605.4642, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12636981904506683, + "rewards/margins": 0.149643212556839, + "rewards/rejected": -0.276013046503067, + "step": 11120 + }, + { + "epoch": 0.73, + "learning_rate": 1.0432691604215695e-06, + "logits/chosen": -2.2541301250457764, + "logits/rejected": -2.172335624694824, + "logps/chosen": -244.57998657226562, + "logps/rejected": -226.08059692382812, + "loss": 1816.2936, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.09533033519983292, + "rewards/margins": 0.1321374773979187, + "rewards/rejected": -0.22746777534484863, + "step": 11130 + }, + { + "epoch": 0.73, + "learning_rate": 1.0386325607759515e-06, + "logits/chosen": -2.268679141998291, + "logits/rejected": -2.209555149078369, + "logps/chosen": -199.47750854492188, + "logps/rejected": -195.68783569335938, + "loss": 1598.9104, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08087374269962311, + "rewards/margins": 0.1628076583147049, + "rewards/rejected": -0.2436813861131668, + "step": 11140 + }, + { + "epoch": 0.73, + "learning_rate": 1.0340035843387544e-06, + "logits/chosen": -2.3822929859161377, + "logits/rejected": -2.054208278656006, + "logps/chosen": -191.0309600830078, + "logps/rejected": -177.69284057617188, + "loss": 1895.2246, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11295398324728012, + "rewards/margins": 0.14244724810123444, + "rewards/rejected": -0.25540122389793396, + "step": 11150 + }, + { + "epoch": 0.73, + "learning_rate": 1.0293822552569887e-06, + "logits/chosen": -2.488461971282959, + "logits/rejected": -2.1910617351531982, + "logps/chosen": -264.654541015625, + "logps/rejected": -224.4275360107422, + "loss": 1848.1217, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.127706840634346, + "rewards/margins": 0.15603476762771606, + "rewards/rejected": -0.2837415933609009, + "step": 11160 + }, + { + "epoch": 0.73, + "learning_rate": 1.0247685976377688e-06, + "logits/chosen": -2.253060817718506, + "logits/rejected": -2.084641695022583, + "logps/chosen": -196.15762329101562, + "logps/rejected": -173.33316040039062, + "loss": 1799.7363, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14472590386867523, + "rewards/margins": 0.16006897389888763, + "rewards/rejected": -0.30479490756988525, + "step": 11170 + }, + { + "epoch": 0.73, + "learning_rate": 1.0201626355481939e-06, + "logits/chosen": -2.4443888664245605, + "logits/rejected": -2.1537623405456543, + "logps/chosen": -229.23739624023438, + "logps/rejected": -192.5585174560547, + "loss": 1444.4172, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.13116887211799622, + "rewards/margins": 0.16881008446216583, + "rewards/rejected": -0.29997897148132324, + "step": 11180 + }, + { + "epoch": 0.73, + "learning_rate": 1.0155643930152192e-06, + "logits/chosen": -2.4378559589385986, + "logits/rejected": -2.357971668243408, + "logps/chosen": -286.31646728515625, + "logps/rejected": -247.3133087158203, + "loss": 1755.0889, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13709110021591187, + "rewards/margins": 0.1564275175333023, + "rewards/rejected": -0.293518602848053, + "step": 11190 + }, + { + "epoch": 0.73, + "learning_rate": 1.0109738940255286e-06, + "logits/chosen": -2.270477056503296, + "logits/rejected": -2.002401351928711, + "logps/chosen": -227.4958953857422, + "logps/rejected": -205.7374267578125, + "loss": 1670.8051, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10513322055339813, + "rewards/margins": 0.15522795915603638, + "rewards/rejected": -0.2603611946105957, + "step": 11200 + }, + { + "epoch": 0.73, + "eval_logits/chosen": -2.339367628097534, + "eval_logits/rejected": -2.152101755142212, + "eval_logps/chosen": -244.8194580078125, + "eval_logps/rejected": -239.89862060546875, + "eval_loss": 1761.804931640625, + "eval_rewards/accuracies": 0.6635000109672546, + "eval_rewards/chosen": -0.12814511358737946, + "eval_rewards/margins": 0.1547221690416336, + "eval_rewards/rejected": -0.28286728262901306, + "eval_runtime": 724.1801, + "eval_samples_per_second": 2.762, + "eval_steps_per_second": 1.381, + "step": 11200 + }, + { + "epoch": 0.73, + "learning_rate": 1.0063911625254155e-06, + "logits/chosen": -2.3362460136413574, + "logits/rejected": -2.194571018218994, + "logps/chosen": -242.2017059326172, + "logps/rejected": -260.4835510253906, + "loss": 1616.4728, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0627903938293457, + "rewards/margins": 0.17652447521686554, + "rewards/rejected": -0.23931488394737244, + "step": 11210 + }, + { + "epoch": 0.73, + "learning_rate": 1.0018162224206502e-06, + "logits/chosen": -2.2498667240142822, + "logits/rejected": -2.1582674980163574, + "logps/chosen": -190.2393341064453, + "logps/rejected": -212.4859161376953, + "loss": 1616.1617, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1576453149318695, + "rewards/margins": 0.18536503612995148, + "rewards/rejected": -0.3430103659629822, + "step": 11220 + }, + { + "epoch": 0.73, + "learning_rate": 9.97249097576363e-07, + "logits/chosen": -2.490760564804077, + "logits/rejected": -2.2085318565368652, + "logps/chosen": -248.25839233398438, + "logps/rejected": -224.3931427001953, + "loss": 1423.3826, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15201839804649353, + "rewards/margins": 0.19626560807228088, + "rewards/rejected": -0.3482840359210968, + "step": 11230 + }, + { + "epoch": 0.74, + "learning_rate": 9.92689811816913e-07, + "logits/chosen": -2.3705766201019287, + "logits/rejected": -2.1356749534606934, + "logps/chosen": -227.1351776123047, + "logps/rejected": -194.965087890625, + "loss": 1767.498, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.14614000916481018, + "rewards/margins": 0.15232881903648376, + "rewards/rejected": -0.29846885800361633, + "step": 11240 + }, + { + "epoch": 0.74, + "learning_rate": 9.881383889257691e-07, + "logits/chosen": -2.307352066040039, + "logits/rejected": -2.3431947231292725, + "logps/chosen": -183.90199279785156, + "logps/rejected": -261.51580810546875, + "loss": 2043.1543, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.14605112373828888, + "rewards/margins": 0.1353534311056137, + "rewards/rejected": -0.2814045548439026, + "step": 11250 + }, + { + "epoch": 0.74, + "learning_rate": 9.835948526453817e-07, + "logits/chosen": -2.1781134605407715, + "logits/rejected": -2.3438239097595215, + "logps/chosen": -197.26766967773438, + "logps/rejected": -252.9156036376953, + "loss": 1805.983, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1683829426765442, + "rewards/margins": 0.11976154148578644, + "rewards/rejected": -0.28814446926116943, + "step": 11260 + }, + { + "epoch": 0.74, + "learning_rate": 9.790592266770633e-07, + "logits/chosen": -2.52354097366333, + "logits/rejected": -2.236750364303589, + "logps/chosen": -280.80609130859375, + "logps/rejected": -268.2882080078125, + "loss": 1571.1008, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11461339890956879, + "rewards/margins": 0.15720847249031067, + "rewards/rejected": -0.27182191610336304, + "step": 11270 + }, + { + "epoch": 0.74, + "learning_rate": 9.745315346808584e-07, + "logits/chosen": -2.21311616897583, + "logits/rejected": -2.0581233501434326, + "logps/chosen": -229.4313201904297, + "logps/rejected": -219.256103515625, + "loss": 1922.0895, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10355854034423828, + "rewards/margins": 0.12207909673452377, + "rewards/rejected": -0.22563764452934265, + "step": 11280 + }, + { + "epoch": 0.74, + "learning_rate": 9.70011800275428e-07, + "logits/chosen": -2.2656772136688232, + "logits/rejected": -2.193603038787842, + "logps/chosen": -253.6857452392578, + "logps/rejected": -281.72552490234375, + "loss": 1775.9613, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16160854697227478, + "rewards/margins": 0.16593758761882782, + "rewards/rejected": -0.3275461792945862, + "step": 11290 + }, + { + "epoch": 0.74, + "learning_rate": 9.655000470379206e-07, + "logits/chosen": -2.1882381439208984, + "logits/rejected": -2.108506679534912, + "logps/chosen": -224.51443481445312, + "logps/rejected": -231.692626953125, + "loss": 1693.2752, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14166946709156036, + "rewards/margins": 0.17860746383666992, + "rewards/rejected": -0.3202769160270691, + "step": 11300 + }, + { + "epoch": 0.74, + "eval_logits/chosen": -2.3482959270477295, + "eval_logits/rejected": -2.1602466106414795, + "eval_logps/chosen": -244.63011169433594, + "eval_logps/rejected": -239.97296142578125, + "eval_loss": 1757.552001953125, + "eval_rewards/accuracies": 0.6650000214576721, + "eval_rewards/chosen": -0.12625156342983246, + "eval_rewards/margins": 0.1573590487241745, + "eval_rewards/rejected": -0.28361061215400696, + "eval_runtime": 722.233, + "eval_samples_per_second": 2.769, + "eval_steps_per_second": 1.385, + "step": 11300 + }, + { + "epoch": 0.74, + "learning_rate": 9.609962985038517e-07, + "logits/chosen": -2.4610273838043213, + "logits/rejected": -2.0612757205963135, + "logps/chosen": -228.94839477539062, + "logps/rejected": -225.9475555419922, + "loss": 1165.8479, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1358734667301178, + "rewards/margins": 0.22124087810516357, + "rewards/rejected": -0.35711434483528137, + "step": 11310 + }, + { + "epoch": 0.74, + "learning_rate": 9.565005781669786e-07, + "logits/chosen": -2.5059690475463867, + "logits/rejected": -2.1450695991516113, + "logps/chosen": -270.55938720703125, + "logps/rejected": -235.69613647460938, + "loss": 1448.8127, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.09968758374452591, + "rewards/margins": 0.18449628353118896, + "rewards/rejected": -0.2841838300228119, + "step": 11320 + }, + { + "epoch": 0.74, + "learning_rate": 9.520129094791822e-07, + "logits/chosen": -2.3138279914855957, + "logits/rejected": -2.1456973552703857, + "logps/chosen": -196.9590301513672, + "logps/rejected": -200.0696258544922, + "loss": 1985.784, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.17463277280330658, + "rewards/margins": 0.13313862681388855, + "rewards/rejected": -0.30777138471603394, + "step": 11330 + }, + { + "epoch": 0.74, + "learning_rate": 9.475333158503389e-07, + "logits/chosen": -2.304564952850342, + "logits/rejected": -2.009404420852661, + "logps/chosen": -224.3502960205078, + "logps/rejected": -195.86581420898438, + "loss": 1605.6322, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.06528019905090332, + "rewards/margins": 0.16674073040485382, + "rewards/rejected": -0.23202089965343475, + "step": 11340 + }, + { + "epoch": 0.74, + "learning_rate": 9.430618206482053e-07, + "logits/chosen": -2.279343843460083, + "logits/rejected": -2.2244575023651123, + "logps/chosen": -149.1036376953125, + "logps/rejected": -159.86256408691406, + "loss": 2122.7781, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.08680722862482071, + "rewards/margins": 0.10163422673940659, + "rewards/rejected": -0.1884414702653885, + "step": 11350 + }, + { + "epoch": 0.74, + "learning_rate": 9.385984471982892e-07, + "logits/chosen": -2.270360231399536, + "logits/rejected": -1.9002015590667725, + "logps/chosen": -225.63143920898438, + "logps/rejected": -198.7368927001953, + "loss": 1430.2678, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1080559641122818, + "rewards/margins": 0.23840036988258362, + "rewards/rejected": -0.3464563190937042, + "step": 11360 + }, + { + "epoch": 0.74, + "learning_rate": 9.341432187837343e-07, + "logits/chosen": -2.315551280975342, + "logits/rejected": -2.2298500537872314, + "logps/chosen": -200.61843872070312, + "logps/rejected": -233.1428680419922, + "loss": 1634.4435, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.10314662754535675, + "rewards/margins": 0.17656059563159943, + "rewards/rejected": -0.27970725297927856, + "step": 11370 + }, + { + "epoch": 0.74, + "learning_rate": 9.29696158645193e-07, + "logits/chosen": -2.2632083892822266, + "logits/rejected": -2.3652756214141846, + "logps/chosen": -232.05648803710938, + "logps/rejected": -283.6464538574219, + "loss": 1160.1338, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10954240709543228, + "rewards/margins": 0.2260010689496994, + "rewards/rejected": -0.33554351329803467, + "step": 11380 + }, + { + "epoch": 0.75, + "learning_rate": 9.252572899807111e-07, + "logits/chosen": -2.323512554168701, + "logits/rejected": -2.353520154953003, + "logps/chosen": -282.01727294921875, + "logps/rejected": -273.61907958984375, + "loss": 1428.1299, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1141771674156189, + "rewards/margins": 0.17788676917552948, + "rewards/rejected": -0.2920639216899872, + "step": 11390 + }, + { + "epoch": 0.75, + "learning_rate": 9.208266359456003e-07, + "logits/chosen": -2.4023215770721436, + "logits/rejected": -2.1396372318267822, + "logps/chosen": -213.10205078125, + "logps/rejected": -237.35195922851562, + "loss": 1789.823, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.07296271622180939, + "rewards/margins": 0.1467919498682022, + "rewards/rejected": -0.2197546660900116, + "step": 11400 + }, + { + "epoch": 0.75, + "eval_logits/chosen": -2.3544914722442627, + "eval_logits/rejected": -2.165926456451416, + "eval_logps/chosen": -243.65545654296875, + "eval_logps/rejected": -239.7627410888672, + "eval_loss": 1758.207275390625, + "eval_rewards/accuracies": 0.6700000166893005, + "eval_rewards/chosen": -0.11650507152080536, + "eval_rewards/margins": 0.16500335931777954, + "eval_rewards/rejected": -0.2815084457397461, + "eval_runtime": 723.912, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.381, + "step": 11400 + }, + { + "epoch": 0.75, + "learning_rate": 9.164042196523229e-07, + "logits/chosen": -2.519958257675171, + "logits/rejected": -2.2136592864990234, + "logps/chosen": -202.1793212890625, + "logps/rejected": -207.7535858154297, + "loss": 1546.1566, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12778282165527344, + "rewards/margins": 0.1909192055463791, + "rewards/rejected": -0.3187020719051361, + "step": 11410 + }, + { + "epoch": 0.75, + "learning_rate": 9.119900641703696e-07, + "logits/chosen": -2.4855868816375732, + "logits/rejected": -2.2158429622650146, + "logps/chosen": -231.71017456054688, + "logps/rejected": -198.54025268554688, + "loss": 1916.1539, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.18263797461986542, + "rewards/margins": 0.13165007531642914, + "rewards/rejected": -0.31428807973861694, + "step": 11420 + }, + { + "epoch": 0.75, + "learning_rate": 9.075841925261364e-07, + "logits/chosen": -2.577622890472412, + "logits/rejected": -2.3363146781921387, + "logps/chosen": -245.819091796875, + "logps/rejected": -251.2655029296875, + "loss": 1526.2436, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07899948209524155, + "rewards/margins": 0.19875195622444153, + "rewards/rejected": -0.2777514159679413, + "step": 11430 + }, + { + "epoch": 0.75, + "learning_rate": 9.031866277028093e-07, + "logits/chosen": -2.2783894538879395, + "logits/rejected": -2.310218095779419, + "logps/chosen": -201.66424560546875, + "logps/rejected": -245.3211669921875, + "loss": 1731.3203, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10589464008808136, + "rewards/margins": 0.1684727668762207, + "rewards/rejected": -0.27436739206314087, + "step": 11440 + }, + { + "epoch": 0.75, + "learning_rate": 8.987973926402391e-07, + "logits/chosen": -2.24288010597229, + "logits/rejected": -2.2750463485717773, + "logps/chosen": -219.508544921875, + "logps/rejected": -234.1844482421875, + "loss": 1930.043, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07168947160243988, + "rewards/margins": 0.15263313055038452, + "rewards/rejected": -0.2243226021528244, + "step": 11450 + }, + { + "epoch": 0.75, + "learning_rate": 8.944165102348273e-07, + "logits/chosen": -2.4590885639190674, + "logits/rejected": -2.307673931121826, + "logps/chosen": -164.0479278564453, + "logps/rejected": -186.90988159179688, + "loss": 1636.5105, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0710080936551094, + "rewards/margins": 0.1560458242893219, + "rewards/rejected": -0.2270539104938507, + "step": 11460 + }, + { + "epoch": 0.75, + "learning_rate": 8.900440033394018e-07, + "logits/chosen": -2.278036594390869, + "logits/rejected": -2.3274552822113037, + "logps/chosen": -192.73841857910156, + "logps/rejected": -194.4464874267578, + "loss": 1998.9707, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.10222195088863373, + "rewards/margins": 0.10993766784667969, + "rewards/rejected": -0.2121596336364746, + "step": 11470 + }, + { + "epoch": 0.75, + "learning_rate": 8.856798947631009e-07, + "logits/chosen": -2.2997279167175293, + "logits/rejected": -2.3228790760040283, + "logps/chosen": -197.69215393066406, + "logps/rejected": -238.3724822998047, + "loss": 1416.2062, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.06816434860229492, + "rewards/margins": 0.23728346824645996, + "rewards/rejected": -0.3054478168487549, + "step": 11480 + }, + { + "epoch": 0.75, + "learning_rate": 8.813242072712519e-07, + "logits/chosen": -2.0630974769592285, + "logits/rejected": -1.944562554359436, + "logps/chosen": -177.80255126953125, + "logps/rejected": -187.6953887939453, + "loss": 2238.0873, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.15940259397029877, + "rewards/margins": 0.08756790310144424, + "rewards/rejected": -0.24697045981884003, + "step": 11490 + }, + { + "epoch": 0.75, + "learning_rate": 8.769769635852557e-07, + "logits/chosen": -2.2842228412628174, + "logits/rejected": -2.3433406352996826, + "logps/chosen": -224.0996856689453, + "logps/rejected": -202.60337829589844, + "loss": 1808.4945, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08791881799697876, + "rewards/margins": 0.15260782837867737, + "rewards/rejected": -0.24052663147449493, + "step": 11500 + }, + { + "epoch": 0.75, + "eval_logits/chosen": -2.357572555541992, + "eval_logits/rejected": -2.168738842010498, + "eval_logps/chosen": -241.220458984375, + "eval_logps/rejected": -236.46685791015625, + "eval_loss": 1761.228271484375, + "eval_rewards/accuracies": 0.6664999723434448, + "eval_rewards/chosen": -0.09215506911277771, + "eval_rewards/margins": 0.1563943475484848, + "eval_rewards/rejected": -0.24854940176010132, + "eval_runtime": 725.395, + "eval_samples_per_second": 2.757, + "eval_steps_per_second": 1.379, + "step": 11500 + }, + { + "epoch": 0.75, + "learning_rate": 8.726381863824635e-07, + "logits/chosen": -2.509023427963257, + "logits/rejected": -2.177807331085205, + "logps/chosen": -294.0090026855469, + "logps/rejected": -243.0245819091797, + "loss": 1735.5961, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.08931988477706909, + "rewards/margins": 0.16421779990196228, + "rewards/rejected": -0.25353771448135376, + "step": 11510 + }, + { + "epoch": 0.75, + "learning_rate": 8.683078982960638e-07, + "logits/chosen": -2.2652676105499268, + "logits/rejected": -1.9574466943740845, + "logps/chosen": -235.6822967529297, + "logps/rejected": -206.75399780273438, + "loss": 1402.3072, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10997311770915985, + "rewards/margins": 0.21162648499011993, + "rewards/rejected": -0.3215996026992798, + "step": 11520 + }, + { + "epoch": 0.75, + "learning_rate": 8.639861219149584e-07, + "logits/chosen": -2.1088919639587402, + "logits/rejected": -2.1795811653137207, + "logps/chosen": -275.09857177734375, + "logps/rejected": -247.9165802001953, + "loss": 1866.335, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11116518825292587, + "rewards/margins": 0.15742537379264832, + "rewards/rejected": -0.2685905694961548, + "step": 11530 + }, + { + "epoch": 0.76, + "learning_rate": 8.596728797836532e-07, + "logits/chosen": -2.247396945953369, + "logits/rejected": -2.0930850505828857, + "logps/chosen": -227.16415405273438, + "logps/rejected": -273.5677795410156, + "loss": 1706.7742, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11855418980121613, + "rewards/margins": 0.1681063324213028, + "rewards/rejected": -0.2866605222225189, + "step": 11540 + }, + { + "epoch": 0.76, + "learning_rate": 8.553681944021294e-07, + "logits/chosen": -2.285822629928589, + "logits/rejected": -2.3091416358947754, + "logps/chosen": -248.54736328125, + "logps/rejected": -248.06689453125, + "loss": 1681.1939, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09463384002447128, + "rewards/margins": 0.18866769969463348, + "rewards/rejected": -0.28330153226852417, + "step": 11550 + }, + { + "epoch": 0.76, + "learning_rate": 8.510720882257365e-07, + "logits/chosen": -2.0751852989196777, + "logits/rejected": -2.222693681716919, + "logps/chosen": -174.25979614257812, + "logps/rejected": -230.66012573242188, + "loss": 1568.4097, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12959204614162445, + "rewards/margins": 0.1516839563846588, + "rewards/rejected": -0.28127604722976685, + "step": 11560 + }, + { + "epoch": 0.76, + "learning_rate": 8.467845836650667e-07, + "logits/chosen": -1.962531328201294, + "logits/rejected": -2.0121817588806152, + "logps/chosen": -220.4276123046875, + "logps/rejected": -242.0430908203125, + "loss": 1793.5357, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1216101422905922, + "rewards/margins": 0.15060429275035858, + "rewards/rejected": -0.27221447229385376, + "step": 11570 + }, + { + "epoch": 0.76, + "learning_rate": 8.425057030858461e-07, + "logits/chosen": -2.1566555500030518, + "logits/rejected": -1.988985300064087, + "logps/chosen": -167.70249938964844, + "logps/rejected": -210.2772979736328, + "loss": 1842.1486, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.11646603047847748, + "rewards/margins": 0.13551898300647736, + "rewards/rejected": -0.2519850432872772, + "step": 11580 + }, + { + "epoch": 0.76, + "learning_rate": 8.382354688088098e-07, + "logits/chosen": -2.3145182132720947, + "logits/rejected": -2.170231580734253, + "logps/chosen": -167.34896850585938, + "logps/rejected": -197.62222290039062, + "loss": 1690.5807, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0919358879327774, + "rewards/margins": 0.14850333333015442, + "rewards/rejected": -0.24043922126293182, + "step": 11590 + }, + { + "epoch": 0.76, + "learning_rate": 8.33973903109594e-07, + "logits/chosen": -2.4262478351593018, + "logits/rejected": -2.16702938079834, + "logps/chosen": -229.05691528320312, + "logps/rejected": -215.0450439453125, + "loss": 1721.7291, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1245979517698288, + "rewards/margins": 0.1754973977804184, + "rewards/rejected": -0.3000953793525696, + "step": 11600 + }, + { + "epoch": 0.76, + "eval_logits/chosen": -2.3435888290405273, + "eval_logits/rejected": -2.1559693813323975, + "eval_logps/chosen": -244.8027801513672, + "eval_logps/rejected": -239.546630859375, + "eval_loss": 1762.330322265625, + "eval_rewards/accuracies": 0.6614999771118164, + "eval_rewards/chosen": -0.12797829508781433, + "eval_rewards/margins": 0.15136903524398804, + "eval_rewards/rejected": -0.27934733033180237, + "eval_runtime": 725.919, + "eval_samples_per_second": 2.755, + "eval_steps_per_second": 1.378, + "step": 11600 + }, + { + "epoch": 0.76, + "learning_rate": 8.297210282186102e-07, + "logits/chosen": -2.189514398574829, + "logits/rejected": -2.1581497192382812, + "logps/chosen": -252.0922088623047, + "logps/rejected": -284.6585388183594, + "loss": 2117.5762, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2207474261522293, + "rewards/margins": 0.11199697107076645, + "rewards/rejected": -0.33274438977241516, + "step": 11610 + }, + { + "epoch": 0.76, + "learning_rate": 8.254768663209397e-07, + "logits/chosen": -2.284914255142212, + "logits/rejected": -2.078519821166992, + "logps/chosen": -289.47747802734375, + "logps/rejected": -241.3529815673828, + "loss": 1649.8689, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10897673666477203, + "rewards/margins": 0.1434708684682846, + "rewards/rejected": -0.25244760513305664, + "step": 11620 + }, + { + "epoch": 0.76, + "learning_rate": 8.212414395562079e-07, + "logits/chosen": -2.1276698112487793, + "logits/rejected": -2.2639830112457275, + "logps/chosen": -247.23141479492188, + "logps/rejected": -283.796875, + "loss": 1854.4729, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17017242312431335, + "rewards/margins": 0.1264459639787674, + "rewards/rejected": -0.29661840200424194, + "step": 11630 + }, + { + "epoch": 0.76, + "learning_rate": 8.170147700184775e-07, + "logits/chosen": -2.3277761936187744, + "logits/rejected": -2.2101144790649414, + "logps/chosen": -268.25653076171875, + "logps/rejected": -278.6501770019531, + "loss": 1479.7791, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.11805230379104614, + "rewards/margins": 0.19326400756835938, + "rewards/rejected": -0.3113163113594055, + "step": 11640 + }, + { + "epoch": 0.76, + "learning_rate": 8.127968797561242e-07, + "logits/chosen": -2.3800370693206787, + "logits/rejected": -2.11928653717041, + "logps/chosen": -241.4968719482422, + "logps/rejected": -239.001953125, + "loss": 1479.4, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17526161670684814, + "rewards/margins": 0.19928061962127686, + "rewards/rejected": -0.374542236328125, + "step": 11650 + }, + { + "epoch": 0.76, + "learning_rate": 8.085877907717338e-07, + "logits/chosen": -2.2742931842803955, + "logits/rejected": -2.2385847568511963, + "logps/chosen": -231.2503204345703, + "logps/rejected": -236.61669921875, + "loss": 1669.0566, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.10338792949914932, + "rewards/margins": 0.17045649886131287, + "rewards/rejected": -0.2738444209098816, + "step": 11660 + }, + { + "epoch": 0.76, + "learning_rate": 8.043875250219732e-07, + "logits/chosen": -2.228330612182617, + "logits/rejected": -2.1692111492156982, + "logps/chosen": -246.3470001220703, + "logps/rejected": -234.47000122070312, + "loss": 1991.2852, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16942265629768372, + "rewards/margins": 0.0919186919927597, + "rewards/rejected": -0.2613413631916046, + "step": 11670 + }, + { + "epoch": 0.76, + "learning_rate": 8.001961044174881e-07, + "logits/chosen": -2.3931221961975098, + "logits/rejected": -2.193916082382202, + "logps/chosen": -243.92312622070312, + "logps/rejected": -198.2577667236328, + "loss": 2045.4062, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.15458671748638153, + "rewards/margins": 0.11522801220417023, + "rewards/rejected": -0.26981475949287415, + "step": 11680 + }, + { + "epoch": 0.76, + "learning_rate": 7.960135508227795e-07, + "logits/chosen": -2.405076503753662, + "logits/rejected": -2.0538601875305176, + "logps/chosen": -308.1584167480469, + "logps/rejected": -259.13482666015625, + "loss": 1822.1654, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14289391040802002, + "rewards/margins": 0.15292389690876007, + "rewards/rejected": -0.2958178222179413, + "step": 11690 + }, + { + "epoch": 0.77, + "learning_rate": 7.91839886056098e-07, + "logits/chosen": -2.420992851257324, + "logits/rejected": -2.168729782104492, + "logps/chosen": -296.9039001464844, + "logps/rejected": -284.8421630859375, + "loss": 1471.2858, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1314167082309723, + "rewards/margins": 0.18508975207805634, + "rewards/rejected": -0.3165064752101898, + "step": 11700 + }, + { + "epoch": 0.77, + "eval_logits/chosen": -2.3449456691741943, + "eval_logits/rejected": -2.157163143157959, + "eval_logps/chosen": -245.83273315429688, + "eval_logps/rejected": -240.5537872314453, + "eval_loss": 1764.1798095703125, + "eval_rewards/accuracies": 0.6610000133514404, + "eval_rewards/chosen": -0.1382778286933899, + "eval_rewards/margins": 0.1511407494544983, + "eval_rewards/rejected": -0.2894185781478882, + "eval_runtime": 723.7931, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.382, + "step": 11700 + }, + { + "epoch": 0.77, + "learning_rate": 7.876751318893217e-07, + "logits/chosen": -2.246511936187744, + "logits/rejected": -1.9968385696411133, + "logps/chosen": -250.1975555419922, + "logps/rejected": -242.7787628173828, + "loss": 1715.1941, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12331908941268921, + "rewards/margins": 0.14169330894947052, + "rewards/rejected": -0.2650124132633209, + "step": 11710 + }, + { + "epoch": 0.77, + "learning_rate": 7.8351931004785e-07, + "logits/chosen": -2.2341372966766357, + "logits/rejected": -1.870633840560913, + "logps/chosen": -224.34896850585938, + "logps/rejected": -214.8955078125, + "loss": 1687.4385, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15720701217651367, + "rewards/margins": 0.15429864823818207, + "rewards/rejected": -0.31150567531585693, + "step": 11720 + }, + { + "epoch": 0.77, + "learning_rate": 7.793724422104834e-07, + "logits/chosen": -2.0860543251037598, + "logits/rejected": -2.245222568511963, + "logps/chosen": -222.11038208007812, + "logps/rejected": -308.7958984375, + "loss": 1587.7459, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1328606903553009, + "rewards/margins": 0.17208316922187805, + "rewards/rejected": -0.30494385957717896, + "step": 11730 + }, + { + "epoch": 0.77, + "learning_rate": 7.752345500093184e-07, + "logits/chosen": -2.3890018463134766, + "logits/rejected": -2.363795757293701, + "logps/chosen": -229.0177764892578, + "logps/rejected": -207.2950439453125, + "loss": 2137.9793, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.18862739205360413, + "rewards/margins": 0.080417200922966, + "rewards/rejected": -0.26904457807540894, + "step": 11740 + }, + { + "epoch": 0.77, + "learning_rate": 7.711056550296253e-07, + "logits/chosen": -2.446387767791748, + "logits/rejected": -2.2817203998565674, + "logps/chosen": -254.32766723632812, + "logps/rejected": -237.6660614013672, + "loss": 1416.5906, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07533854246139526, + "rewards/margins": 0.20363454520702362, + "rewards/rejected": -0.2789730727672577, + "step": 11750 + }, + { + "epoch": 0.77, + "learning_rate": 7.669857788097445e-07, + "logits/chosen": -2.1872541904449463, + "logits/rejected": -1.9422680139541626, + "logps/chosen": -177.9638671875, + "logps/rejected": -220.3321075439453, + "loss": 1753.0037, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.16397897899150848, + "rewards/margins": 0.15743349492549896, + "rewards/rejected": -0.32141250371932983, + "step": 11760 + }, + { + "epoch": 0.77, + "learning_rate": 7.628749428409676e-07, + "logits/chosen": -2.4421238899230957, + "logits/rejected": -2.0218122005462646, + "logps/chosen": -252.5674285888672, + "logps/rejected": -207.2790069580078, + "loss": 2211.9254, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15679652988910675, + "rewards/margins": 0.13463090360164642, + "rewards/rejected": -0.2914274036884308, + "step": 11770 + }, + { + "epoch": 0.77, + "learning_rate": 7.587731685674288e-07, + "logits/chosen": -2.3191511631011963, + "logits/rejected": -2.3603625297546387, + "logps/chosen": -286.06329345703125, + "logps/rejected": -306.73150634765625, + "loss": 1985.7795, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12475011497735977, + "rewards/margins": 0.12831243872642517, + "rewards/rejected": -0.25306254625320435, + "step": 11780 + }, + { + "epoch": 0.77, + "learning_rate": 7.546804773859931e-07, + "logits/chosen": -2.428969144821167, + "logits/rejected": -2.1997106075286865, + "logps/chosen": -239.6063232421875, + "logps/rejected": -237.9988555908203, + "loss": 1582.8656, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12566278874874115, + "rewards/margins": 0.18328523635864258, + "rewards/rejected": -0.30894801020622253, + "step": 11790 + }, + { + "epoch": 0.77, + "learning_rate": 7.505968906461409e-07, + "logits/chosen": -2.335308074951172, + "logits/rejected": -2.1804909706115723, + "logps/chosen": -255.17919921875, + "logps/rejected": -248.5977325439453, + "loss": 1792.252, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14239093661308289, + "rewards/margins": 0.17564840614795685, + "rewards/rejected": -0.3180393576622009, + "step": 11800 + }, + { + "epoch": 0.77, + "eval_logits/chosen": -2.356790781021118, + "eval_logits/rejected": -2.1682629585266113, + "eval_logps/chosen": -244.12054443359375, + "eval_logps/rejected": -239.302001953125, + "eval_loss": 1759.88671875, + "eval_rewards/accuracies": 0.6669999957084656, + "eval_rewards/chosen": -0.1211560070514679, + "eval_rewards/margins": 0.15574483573436737, + "eval_rewards/rejected": -0.27690085768699646, + "eval_runtime": 724.6407, + "eval_samples_per_second": 2.76, + "eval_steps_per_second": 1.38, + "step": 11800 + }, + { + "epoch": 0.77, + "learning_rate": 7.465224296498627e-07, + "logits/chosen": -2.4071779251098633, + "logits/rejected": -2.0205078125, + "logps/chosen": -243.91873168945312, + "logps/rejected": -220.3428497314453, + "loss": 1434.0618, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10959659516811371, + "rewards/margins": 0.17553094029426575, + "rewards/rejected": -0.28512755036354065, + "step": 11810 + }, + { + "epoch": 0.77, + "learning_rate": 7.424571156515412e-07, + "logits/chosen": -2.2748305797576904, + "logits/rejected": -2.2719333171844482, + "logps/chosen": -189.68267822265625, + "logps/rejected": -227.21694946289062, + "loss": 1762.3602, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11977501958608627, + "rewards/margins": 0.14857666194438934, + "rewards/rejected": -0.268351674079895, + "step": 11820 + }, + { + "epoch": 0.77, + "learning_rate": 7.38400969857847e-07, + "logits/chosen": -2.2314724922180176, + "logits/rejected": -2.01545786857605, + "logps/chosen": -202.58847045898438, + "logps/rejected": -223.5933837890625, + "loss": 1383.0405, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.19446060061454773, + "rewards/margins": 0.17852742969989777, + "rewards/rejected": -0.3729880452156067, + "step": 11830 + }, + { + "epoch": 0.77, + "learning_rate": 7.343540134276225e-07, + "logits/chosen": -2.3522350788116455, + "logits/rejected": -2.277493953704834, + "logps/chosen": -180.2784881591797, + "logps/rejected": -196.76797485351562, + "loss": 1910.5359, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.09566868841648102, + "rewards/margins": 0.1459321677684784, + "rewards/rejected": -0.24160084128379822, + "step": 11840 + }, + { + "epoch": 0.78, + "learning_rate": 7.303162674717762e-07, + "logits/chosen": -2.3356125354766846, + "logits/rejected": -1.941916823387146, + "logps/chosen": -226.21249389648438, + "logps/rejected": -181.681640625, + "loss": 2040.407, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15984733402729034, + "rewards/margins": 0.11304545402526855, + "rewards/rejected": -0.2728927731513977, + "step": 11850 + }, + { + "epoch": 0.78, + "learning_rate": 7.26287753053167e-07, + "logits/chosen": -2.286027431488037, + "logits/rejected": -2.2168824672698975, + "logps/chosen": -282.25958251953125, + "logps/rejected": -296.0490417480469, + "loss": 1808.9691, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1728345900774002, + "rewards/margins": 0.15057656168937683, + "rewards/rejected": -0.32341113686561584, + "step": 11860 + }, + { + "epoch": 0.78, + "learning_rate": 7.222684911865013e-07, + "logits/chosen": -2.3952112197875977, + "logits/rejected": -2.4059479236602783, + "logps/chosen": -214.5403289794922, + "logps/rejected": -256.82745361328125, + "loss": 1377.1188, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.07371652126312256, + "rewards/margins": 0.20601694285869598, + "rewards/rejected": -0.27973347902297974, + "step": 11870 + }, + { + "epoch": 0.78, + "learning_rate": 7.182585028382166e-07, + "logits/chosen": -2.44596266746521, + "logits/rejected": -2.1434316635131836, + "logps/chosen": -281.9929504394531, + "logps/rejected": -267.156982421875, + "loss": 1530.4185, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.05841357633471489, + "rewards/margins": 0.17613962292671204, + "rewards/rejected": -0.23455317318439484, + "step": 11880 + }, + { + "epoch": 0.78, + "learning_rate": 7.142578089263769e-07, + "logits/chosen": -2.4750990867614746, + "logits/rejected": -2.1455719470977783, + "logps/chosen": -327.4580078125, + "logps/rejected": -288.38934326171875, + "loss": 1571.5391, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09029410779476166, + "rewards/margins": 0.19731321930885315, + "rewards/rejected": -0.287607342004776, + "step": 11890 + }, + { + "epoch": 0.78, + "learning_rate": 7.102664303205611e-07, + "logits/chosen": -2.355468511581421, + "logits/rejected": -2.102905750274658, + "logps/chosen": -234.4264373779297, + "logps/rejected": -229.1504669189453, + "loss": 2080.723, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1386391818523407, + "rewards/margins": 0.1260056495666504, + "rewards/rejected": -0.2646448314189911, + "step": 11900 + }, + { + "epoch": 0.78, + "eval_logits/chosen": -2.3557183742523193, + "eval_logits/rejected": -2.1673061847686768, + "eval_logps/chosen": -243.10972595214844, + "eval_logps/rejected": -238.54348754882812, + "eval_loss": 1758.6500244140625, + "eval_rewards/accuracies": 0.6664999723434448, + "eval_rewards/chosen": -0.11104759573936462, + "eval_rewards/margins": 0.15826837718486786, + "eval_rewards/rejected": -0.2693159878253937, + "eval_runtime": 725.2021, + "eval_samples_per_second": 2.758, + "eval_steps_per_second": 1.379, + "step": 11900 + }, + { + "epoch": 0.78, + "learning_rate": 7.062843878417566e-07, + "logits/chosen": -2.463575839996338, + "logits/rejected": -2.331946849822998, + "logps/chosen": -232.64248657226562, + "logps/rejected": -213.59414672851562, + "loss": 1912.4789, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09209892153739929, + "rewards/margins": 0.13995857536792755, + "rewards/rejected": -0.23205752670764923, + "step": 11910 + }, + { + "epoch": 0.78, + "learning_rate": 7.023117022622458e-07, + "logits/chosen": -2.394233226776123, + "logits/rejected": -2.0388636589050293, + "logps/chosen": -254.5091094970703, + "logps/rejected": -243.8262481689453, + "loss": 1850.1949, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15993770956993103, + "rewards/margins": 0.12585516273975372, + "rewards/rejected": -0.28579288721084595, + "step": 11920 + }, + { + "epoch": 0.78, + "learning_rate": 6.983483943055042e-07, + "logits/chosen": -2.2799057960510254, + "logits/rejected": -2.101398229598999, + "logps/chosen": -290.40399169921875, + "logps/rejected": -246.39370727539062, + "loss": 1913.1725, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.09937360137701035, + "rewards/margins": 0.12762802839279175, + "rewards/rejected": -0.2270016223192215, + "step": 11930 + }, + { + "epoch": 0.78, + "learning_rate": 6.943944846460859e-07, + "logits/chosen": -2.3366754055023193, + "logits/rejected": -2.2716286182403564, + "logps/chosen": -228.8292694091797, + "logps/rejected": -195.3153839111328, + "loss": 2120.9512, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.10074935853481293, + "rewards/margins": 0.13228239119052887, + "rewards/rejected": -0.2330317497253418, + "step": 11940 + }, + { + "epoch": 0.78, + "learning_rate": 6.904499939095225e-07, + "logits/chosen": -2.298487663269043, + "logits/rejected": -2.2476203441619873, + "logps/chosen": -228.70315551757812, + "logps/rejected": -232.03164672851562, + "loss": 1719.4844, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.06355978548526764, + "rewards/margins": 0.18656757473945618, + "rewards/rejected": -0.250127375125885, + "step": 11950 + }, + { + "epoch": 0.78, + "learning_rate": 6.865149426722079e-07, + "logits/chosen": -2.273329257965088, + "logits/rejected": -2.219085454940796, + "logps/chosen": -285.75531005859375, + "logps/rejected": -265.9485168457031, + "loss": 2047.2496, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14570143818855286, + "rewards/margins": 0.10887260735034943, + "rewards/rejected": -0.2545740604400635, + "step": 11960 + }, + { + "epoch": 0.78, + "learning_rate": 6.825893514612985e-07, + "logits/chosen": -2.1099815368652344, + "logits/rejected": -2.2191219329833984, + "logps/chosen": -242.7589111328125, + "logps/rejected": -258.30767822265625, + "loss": 1666.5125, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.061920829117298126, + "rewards/margins": 0.17732374370098114, + "rewards/rejected": -0.23924458026885986, + "step": 11970 + }, + { + "epoch": 0.78, + "learning_rate": 6.786732407546001e-07, + "logits/chosen": -2.126340866088867, + "logits/rejected": -2.080836772918701, + "logps/chosen": -208.6787109375, + "logps/rejected": -183.4300994873047, + "loss": 1800.6842, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07797534018754959, + "rewards/margins": 0.14466950297355652, + "rewards/rejected": -0.2226448357105255, + "step": 11980 + }, + { + "epoch": 0.78, + "learning_rate": 6.747666309804654e-07, + "logits/chosen": -2.5057218074798584, + "logits/rejected": -2.184866189956665, + "logps/chosen": -296.4242858886719, + "logps/rejected": -236.760498046875, + "loss": 1271.4783, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08747060596942902, + "rewards/margins": 0.1987169086933136, + "rewards/rejected": -0.2861874997615814, + "step": 11990 + }, + { + "epoch": 0.79, + "learning_rate": 6.708695425176831e-07, + "logits/chosen": -2.1408398151397705, + "logits/rejected": -2.1481544971466064, + "logps/chosen": -179.84515380859375, + "logps/rejected": -217.2546844482422, + "loss": 1576.5392, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10806410014629364, + "rewards/margins": 0.16654148697853088, + "rewards/rejected": -0.2746056020259857, + "step": 12000 + }, + { + "epoch": 0.79, + "eval_logits/chosen": -2.3568789958953857, + "eval_logits/rejected": -2.1684751510620117, + "eval_logps/chosen": -240.02801513671875, + "eval_logps/rejected": -235.32835388183594, + "eval_loss": 1760.51953125, + "eval_rewards/accuracies": 0.6669999957084656, + "eval_rewards/chosen": -0.08023066818714142, + "eval_rewards/margins": 0.15693405270576477, + "eval_rewards/rejected": -0.237164705991745, + "eval_runtime": 723.4723, + "eval_samples_per_second": 2.764, + "eval_steps_per_second": 1.382, + "step": 12000 + }, + { + "epoch": 0.79, + "learning_rate": 6.669819956953768e-07, + "logits/chosen": -2.1999356746673584, + "logits/rejected": -2.1322684288024902, + "logps/chosen": -176.7624969482422, + "logps/rejected": -200.78819274902344, + "loss": 1586.7117, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.06431283056735992, + "rewards/margins": 0.15485699474811554, + "rewards/rejected": -0.21916981041431427, + "step": 12010 + }, + { + "epoch": 0.79, + "learning_rate": 6.631040107928957e-07, + "logits/chosen": -2.527143955230713, + "logits/rejected": -2.213205337524414, + "logps/chosen": -275.36004638671875, + "logps/rejected": -196.37973022460938, + "loss": 1398.0136, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07175728678703308, + "rewards/margins": 0.17318230867385864, + "rewards/rejected": -0.24493959546089172, + "step": 12020 + }, + { + "epoch": 0.79, + "learning_rate": 6.592356080397072e-07, + "logits/chosen": -2.4240286350250244, + "logits/rejected": -1.8601045608520508, + "logps/chosen": -234.77880859375, + "logps/rejected": -196.73764038085938, + "loss": 1555.797, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.06489615887403488, + "rewards/margins": 0.18422222137451172, + "rewards/rejected": -0.2491184026002884, + "step": 12030 + }, + { + "epoch": 0.79, + "learning_rate": 6.553768076152963e-07, + "logits/chosen": -2.3208096027374268, + "logits/rejected": -2.4190030097961426, + "logps/chosen": -164.54904174804688, + "logps/rejected": -212.4931182861328, + "loss": 1571.5309, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.0645146369934082, + "rewards/margins": 0.20840032398700714, + "rewards/rejected": -0.27291494607925415, + "step": 12040 + }, + { + "epoch": 0.79, + "learning_rate": 6.51527629649055e-07, + "logits/chosen": -2.4525504112243652, + "logits/rejected": -2.3045573234558105, + "logps/chosen": -265.19659423828125, + "logps/rejected": -245.7611846923828, + "loss": 2120.6713, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.11613516509532928, + "rewards/margins": 0.09711629152297974, + "rewards/rejected": -0.21325147151947021, + "step": 12050 + }, + { + "epoch": 0.79, + "learning_rate": 6.476880942201824e-07, + "logits/chosen": -2.583015203475952, + "logits/rejected": -2.1839451789855957, + "logps/chosen": -243.401123046875, + "logps/rejected": -204.27493286132812, + "loss": 1704.0209, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.020634567365050316, + "rewards/margins": 0.19498001039028168, + "rewards/rejected": -0.21561458706855774, + "step": 12060 + }, + { + "epoch": 0.79, + "learning_rate": 6.438582213575748e-07, + "logits/chosen": -2.275463819503784, + "logits/rejected": -2.2460455894470215, + "logps/chosen": -239.5929718017578, + "logps/rejected": -266.6346740722656, + "loss": 1807.3643, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09676406532526016, + "rewards/margins": 0.15896211564540863, + "rewards/rejected": -0.2557261884212494, + "step": 12070 + }, + { + "epoch": 0.79, + "learning_rate": 6.400380310397267e-07, + "logits/chosen": -2.2111427783966064, + "logits/rejected": -2.2207419872283936, + "logps/chosen": -243.8424835205078, + "logps/rejected": -285.3353271484375, + "loss": 2440.1971, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.11732114851474762, + "rewards/margins": 0.10081054270267487, + "rewards/rejected": -0.21813169121742249, + "step": 12080 + }, + { + "epoch": 0.79, + "learning_rate": 6.362275431946202e-07, + "logits/chosen": -2.1953682899475098, + "logits/rejected": -2.2317399978637695, + "logps/chosen": -245.20437622070312, + "logps/rejected": -265.2196960449219, + "loss": 1881.7752, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.08850773423910141, + "rewards/margins": 0.13967140018939972, + "rewards/rejected": -0.22817914187908173, + "step": 12090 + }, + { + "epoch": 0.79, + "learning_rate": 6.324267776996285e-07, + "logits/chosen": -2.4136674404144287, + "logits/rejected": -2.035423755645752, + "logps/chosen": -382.706298828125, + "logps/rejected": -284.49853515625, + "loss": 1299.8462, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.08127235621213913, + "rewards/margins": 0.23772704601287842, + "rewards/rejected": -0.31899940967559814, + "step": 12100 + }, + { + "epoch": 0.79, + "eval_logits/chosen": -2.3554248809814453, + "eval_logits/rejected": -2.1673924922943115, + "eval_logps/chosen": -239.875, + "eval_logps/rejected": -234.68032836914062, + "eval_loss": 1767.3917236328125, + "eval_rewards/accuracies": 0.6579999923706055, + "eval_rewards/chosen": -0.07870057225227356, + "eval_rewards/margins": 0.15198378264904022, + "eval_rewards/rejected": -0.23068435490131378, + "eval_runtime": 725.0964, + "eval_samples_per_second": 2.758, + "eval_steps_per_second": 1.379, + "step": 12100 + }, + { + "epoch": 0.79, + "learning_rate": 6.286357543814045e-07, + "logits/chosen": -2.2756481170654297, + "logits/rejected": -2.1830382347106934, + "logps/chosen": -200.79925537109375, + "logps/rejected": -285.17669677734375, + "loss": 1763.618, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.07269416749477386, + "rewards/margins": 0.1636621505022049, + "rewards/rejected": -0.23635630309581757, + "step": 12110 + }, + { + "epoch": 0.79, + "learning_rate": 6.248544930157838e-07, + "logits/chosen": -2.4011549949645996, + "logits/rejected": -2.153775691986084, + "logps/chosen": -191.3563232421875, + "logps/rejected": -194.35769653320312, + "loss": 1513.909, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07270742952823639, + "rewards/margins": 0.2078186273574829, + "rewards/rejected": -0.2805260419845581, + "step": 12120 + }, + { + "epoch": 0.79, + "learning_rate": 6.21083013327678e-07, + "logits/chosen": -2.332559108734131, + "logits/rejected": -2.230076789855957, + "logps/chosen": -309.12188720703125, + "logps/rejected": -265.01318359375, + "loss": 1938.6658, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.04881126806139946, + "rewards/margins": 0.11817065626382828, + "rewards/rejected": -0.16698193550109863, + "step": 12130 + }, + { + "epoch": 0.79, + "learning_rate": 6.17321334990973e-07, + "logits/chosen": -2.297025203704834, + "logits/rejected": -2.2214016914367676, + "logps/chosen": -207.6863250732422, + "logps/rejected": -186.2632598876953, + "loss": 1931.7465, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.06881825625896454, + "rewards/margins": 0.13814252614974976, + "rewards/rejected": -0.2069607973098755, + "step": 12140 + }, + { + "epoch": 0.79, + "learning_rate": 6.135694776284243e-07, + "logits/chosen": -2.4541053771972656, + "logits/rejected": -2.192342519760132, + "logps/chosen": -277.95233154296875, + "logps/rejected": -232.65432739257812, + "loss": 1577.0223, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07161318510770798, + "rewards/margins": 0.19474197924137115, + "rewards/rejected": -0.2663551867008209, + "step": 12150 + }, + { + "epoch": 0.8, + "learning_rate": 6.098274608115595e-07, + "logits/chosen": -2.2446258068084717, + "logits/rejected": -2.1158833503723145, + "logps/chosen": -211.1138153076172, + "logps/rejected": -193.41287231445312, + "loss": 1944.4771, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.05463189631700516, + "rewards/margins": 0.10931450128555298, + "rewards/rejected": -0.16394639015197754, + "step": 12160 + }, + { + "epoch": 0.8, + "learning_rate": 6.060953040605697e-07, + "logits/chosen": -2.464473247528076, + "logits/rejected": -1.9369499683380127, + "logps/chosen": -342.2126770019531, + "logps/rejected": -296.26025390625, + "loss": 1398.6549, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.030829787254333496, + "rewards/margins": 0.2394247055053711, + "rewards/rejected": -0.2702544629573822, + "step": 12170 + }, + { + "epoch": 0.8, + "learning_rate": 6.023730268442144e-07, + "logits/chosen": -2.2708048820495605, + "logits/rejected": -2.09238338470459, + "logps/chosen": -212.4055938720703, + "logps/rejected": -203.7031707763672, + "loss": 1372.8158, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07584762573242188, + "rewards/margins": 0.19742408394813538, + "rewards/rejected": -0.27327170968055725, + "step": 12180 + }, + { + "epoch": 0.8, + "learning_rate": 5.986606485797131e-07, + "logits/chosen": -2.2674381732940674, + "logits/rejected": -2.050218105316162, + "logps/chosen": -212.3667449951172, + "logps/rejected": -229.31253051757812, + "loss": 1873.1357, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.12476187944412231, + "rewards/margins": 0.11879698187112808, + "rewards/rejected": -0.2435588836669922, + "step": 12190 + }, + { + "epoch": 0.8, + "learning_rate": 5.949581886326511e-07, + "logits/chosen": -2.3818600177764893, + "logits/rejected": -2.3597679138183594, + "logps/chosen": -304.85516357421875, + "logps/rejected": -290.83856201171875, + "loss": 1648.0504, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07063116133213043, + "rewards/margins": 0.1489768922328949, + "rewards/rejected": -0.21960802376270294, + "step": 12200 + }, + { + "epoch": 0.8, + "eval_logits/chosen": -2.353001356124878, + "eval_logits/rejected": -2.1653261184692383, + "eval_logps/chosen": -240.8780517578125, + "eval_logps/rejected": -235.7217254638672, + "eval_loss": 1765.9278564453125, + "eval_rewards/accuracies": 0.6600000262260437, + "eval_rewards/chosen": -0.08873096108436584, + "eval_rewards/margins": 0.15236733853816986, + "eval_rewards/rejected": -0.2410982847213745, + "eval_runtime": 722.6815, + "eval_samples_per_second": 2.767, + "eval_steps_per_second": 1.384, + "step": 12200 + }, + { + "epoch": 0.8, + "learning_rate": 5.912656663168717e-07, + "logits/chosen": -2.436596632003784, + "logits/rejected": -2.380277633666992, + "logps/chosen": -232.26504516601562, + "logps/rejected": -241.3466033935547, + "loss": 1602.81, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.0825573205947876, + "rewards/margins": 0.15453985333442688, + "rewards/rejected": -0.23709718883037567, + "step": 12210 + }, + { + "epoch": 0.8, + "learning_rate": 5.875831008943817e-07, + "logits/chosen": -2.193098545074463, + "logits/rejected": -2.1777737140655518, + "logps/chosen": -185.7943878173828, + "logps/rejected": -174.64498901367188, + "loss": 2011.7496, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.08150316029787064, + "rewards/margins": 0.1145082339644432, + "rewards/rejected": -0.19601139426231384, + "step": 12220 + }, + { + "epoch": 0.8, + "learning_rate": 5.839105115752442e-07, + "logits/chosen": -2.3216700553894043, + "logits/rejected": -2.134594440460205, + "logps/chosen": -237.3351593017578, + "logps/rejected": -210.3347625732422, + "loss": 2003.9154, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.14357665181159973, + "rewards/margins": 0.1300736516714096, + "rewards/rejected": -0.27365028858184814, + "step": 12230 + }, + { + "epoch": 0.8, + "learning_rate": 5.802479175174855e-07, + "logits/chosen": -2.300666332244873, + "logits/rejected": -2.1491894721984863, + "logps/chosen": -175.96163940429688, + "logps/rejected": -196.45407104492188, + "loss": 1881.6727, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.09967365860939026, + "rewards/margins": 0.14565333724021912, + "rewards/rejected": -0.245326966047287, + "step": 12240 + }, + { + "epoch": 0.8, + "learning_rate": 5.765953378269901e-07, + "logits/chosen": -2.2157773971557617, + "logits/rejected": -2.150618553161621, + "logps/chosen": -216.1068572998047, + "logps/rejected": -263.0898742675781, + "loss": 1716.0438, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09113435447216034, + "rewards/margins": 0.15939262509346008, + "rewards/rejected": -0.2505269944667816, + "step": 12250 + }, + { + "epoch": 0.8, + "learning_rate": 5.729527915574037e-07, + "logits/chosen": -2.3772082328796387, + "logits/rejected": -2.235239267349243, + "logps/chosen": -229.6256561279297, + "logps/rejected": -246.18142700195312, + "loss": 1565.1001, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09207095205783844, + "rewards/margins": 0.16216661036014557, + "rewards/rejected": -0.254237562417984, + "step": 12260 + }, + { + "epoch": 0.8, + "learning_rate": 5.693202977100304e-07, + "logits/chosen": -2.3882689476013184, + "logits/rejected": -2.0996174812316895, + "logps/chosen": -181.91680908203125, + "logps/rejected": -189.93724060058594, + "loss": 2011.5711, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11771687120199203, + "rewards/margins": 0.11356747150421143, + "rewards/rejected": -0.23128433525562286, + "step": 12270 + }, + { + "epoch": 0.8, + "learning_rate": 5.656978752337389e-07, + "logits/chosen": -2.4091591835021973, + "logits/rejected": -2.2006168365478516, + "logps/chosen": -211.9903564453125, + "logps/rejected": -229.5045166015625, + "loss": 1806.8143, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12335234880447388, + "rewards/margins": 0.16114404797554016, + "rewards/rejected": -0.2844964265823364, + "step": 12280 + }, + { + "epoch": 0.8, + "learning_rate": 5.620855430248581e-07, + "logits/chosen": -2.3077521324157715, + "logits/rejected": -2.163949489593506, + "logps/chosen": -165.94668579101562, + "logps/rejected": -183.89791870117188, + "loss": 1440.0475, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.039516277611255646, + "rewards/margins": 0.20129624009132385, + "rewards/rejected": -0.2408125102519989, + "step": 12290 + }, + { + "epoch": 0.8, + "learning_rate": 5.584833199270837e-07, + "logits/chosen": -2.365527868270874, + "logits/rejected": -2.222726345062256, + "logps/chosen": -239.72311401367188, + "logps/rejected": -245.8061065673828, + "loss": 1886.5662, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11090850830078125, + "rewards/margins": 0.13776203989982605, + "rewards/rejected": -0.2486705780029297, + "step": 12300 + }, + { + "epoch": 0.8, + "eval_logits/chosen": -2.353067398071289, + "eval_logits/rejected": -2.165477752685547, + "eval_logps/chosen": -241.03871154785156, + "eval_logps/rejected": -235.90528869628906, + "eval_loss": 1764.2982177734375, + "eval_rewards/accuracies": 0.6614999771118164, + "eval_rewards/chosen": -0.0903375893831253, + "eval_rewards/margins": 0.15259619057178497, + "eval_rewards/rejected": -0.24293380975723267, + "eval_runtime": 724.9294, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.379, + "step": 12300 + }, + { + "epoch": 0.81, + "learning_rate": 5.548912247313742e-07, + "logits/chosen": -2.5742082595825195, + "logits/rejected": -2.145359516143799, + "logps/chosen": -295.21185302734375, + "logps/rejected": -260.882568359375, + "loss": 1808.2764, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1141192689538002, + "rewards/margins": 0.13246312737464905, + "rewards/rejected": -0.24658238887786865, + "step": 12310 + }, + { + "epoch": 0.81, + "learning_rate": 5.513092761758596e-07, + "logits/chosen": -2.3879361152648926, + "logits/rejected": -2.1963579654693604, + "logps/chosen": -284.59625244140625, + "logps/rejected": -227.46878051757812, + "loss": 2101.8111, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13386356830596924, + "rewards/margins": 0.09853416681289673, + "rewards/rejected": -0.23239775002002716, + "step": 12320 + }, + { + "epoch": 0.81, + "learning_rate": 5.477374929457363e-07, + "logits/chosen": -2.3187971115112305, + "logits/rejected": -2.330688238143921, + "logps/chosen": -212.56417846679688, + "logps/rejected": -205.9453125, + "loss": 1889.5531, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10503029823303223, + "rewards/margins": 0.12553009390830994, + "rewards/rejected": -0.23056037724018097, + "step": 12330 + }, + { + "epoch": 0.81, + "learning_rate": 5.441758936731772e-07, + "logits/chosen": -2.337404489517212, + "logits/rejected": -2.228415012359619, + "logps/chosen": -247.33132934570312, + "logps/rejected": -244.2796630859375, + "loss": 1912.2846, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.11986031383275986, + "rewards/margins": 0.1473861038684845, + "rewards/rejected": -0.26724642515182495, + "step": 12340 + }, + { + "epoch": 0.81, + "learning_rate": 5.406244969372273e-07, + "logits/chosen": -2.252225399017334, + "logits/rejected": -2.09260892868042, + "logps/chosen": -209.585693359375, + "logps/rejected": -238.00643920898438, + "loss": 1337.8583, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09862855821847916, + "rewards/margins": 0.21918770670890808, + "rewards/rejected": -0.31781625747680664, + "step": 12350 + }, + { + "epoch": 0.81, + "learning_rate": 5.370833212637122e-07, + "logits/chosen": -2.320801258087158, + "logits/rejected": -2.0274157524108887, + "logps/chosen": -230.2029266357422, + "logps/rejected": -232.7014923095703, + "loss": 1679.7021, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11691944301128387, + "rewards/margins": 0.16283735632896423, + "rewards/rejected": -0.2797567844390869, + "step": 12360 + }, + { + "epoch": 0.81, + "learning_rate": 5.335523851251392e-07, + "logits/chosen": -2.2678158283233643, + "logits/rejected": -2.1951239109039307, + "logps/chosen": -214.69039916992188, + "logps/rejected": -208.9894561767578, + "loss": 2124.7422, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.08556528389453888, + "rewards/margins": 0.15097244083881378, + "rewards/rejected": -0.23653773963451385, + "step": 12370 + }, + { + "epoch": 0.81, + "learning_rate": 5.300317069406003e-07, + "logits/chosen": -2.244819164276123, + "logits/rejected": -2.2248756885528564, + "logps/chosen": -170.40798950195312, + "logps/rejected": -190.18472290039062, + "loss": 1655.8439, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.09662125259637833, + "rewards/margins": 0.1600993573665619, + "rewards/rejected": -0.2567206025123596, + "step": 12380 + }, + { + "epoch": 0.81, + "learning_rate": 5.265213050756782e-07, + "logits/chosen": -2.4712936878204346, + "logits/rejected": -2.2991206645965576, + "logps/chosen": -242.697021484375, + "logps/rejected": -253.659423828125, + "loss": 2287.0518, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.10645761340856552, + "rewards/margins": 0.13093502819538116, + "rewards/rejected": -0.2373926341533661, + "step": 12390 + }, + { + "epoch": 0.81, + "learning_rate": 5.230211978423477e-07, + "logits/chosen": -2.3769500255584717, + "logits/rejected": -2.260720729827881, + "logps/chosen": -230.4022979736328, + "logps/rejected": -230.37966918945312, + "loss": 1838.3824, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11602053791284561, + "rewards/margins": 0.15146169066429138, + "rewards/rejected": -0.2674822211265564, + "step": 12400 + }, + { + "epoch": 0.81, + "eval_logits/chosen": -2.3548293113708496, + "eval_logits/rejected": -2.167078971862793, + "eval_logps/chosen": -241.44473266601562, + "eval_logps/rejected": -235.86338806152344, + "eval_loss": 1764.401611328125, + "eval_rewards/accuracies": 0.6610000133514404, + "eval_rewards/chosen": -0.09439782798290253, + "eval_rewards/margins": 0.14811697602272034, + "eval_rewards/rejected": -0.24251480400562286, + "eval_runtime": 724.5089, + "eval_samples_per_second": 2.76, + "eval_steps_per_second": 1.38, + "step": 12400 + }, + { + "epoch": 0.81, + "learning_rate": 5.195314034988835e-07, + "logits/chosen": -2.501450300216675, + "logits/rejected": -2.2392566204071045, + "logps/chosen": -226.6898193359375, + "logps/rejected": -175.55081176757812, + "loss": 1677.6783, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.07402725517749786, + "rewards/margins": 0.15900227427482605, + "rewards/rejected": -0.23302951455116272, + "step": 12410 + }, + { + "epoch": 0.81, + "learning_rate": 5.160519402497616e-07, + "logits/chosen": -2.389047622680664, + "logits/rejected": -2.2399632930755615, + "logps/chosen": -243.2234344482422, + "logps/rejected": -253.6158905029297, + "loss": 1699.8174, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13169410824775696, + "rewards/margins": 0.14452390372753143, + "rewards/rejected": -0.2762180268764496, + "step": 12420 + }, + { + "epoch": 0.81, + "learning_rate": 5.125828262455679e-07, + "logits/chosen": -2.281661033630371, + "logits/rejected": -2.0835673809051514, + "logps/chosen": -263.7641906738281, + "logps/rejected": -250.3699188232422, + "loss": 1599.9329, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.08051087707281113, + "rewards/margins": 0.16945312917232513, + "rewards/rejected": -0.24996399879455566, + "step": 12430 + }, + { + "epoch": 0.81, + "learning_rate": 5.091240795828992e-07, + "logits/chosen": -2.0497326850891113, + "logits/rejected": -2.242506980895996, + "logps/chosen": -211.437744140625, + "logps/rejected": -249.03170776367188, + "loss": 1446.9986, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08947314321994781, + "rewards/margins": 0.17763306200504303, + "rewards/rejected": -0.26710623502731323, + "step": 12440 + }, + { + "epoch": 0.81, + "learning_rate": 5.056757183042732e-07, + "logits/chosen": -2.2784876823425293, + "logits/rejected": -2.1990206241607666, + "logps/chosen": -245.272705078125, + "logps/rejected": -241.74923706054688, + "loss": 1716.6078, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11716543138027191, + "rewards/margins": 0.1635403037071228, + "rewards/rejected": -0.2807057499885559, + "step": 12450 + }, + { + "epoch": 0.82, + "learning_rate": 5.022377603980308e-07, + "logits/chosen": -2.4438557624816895, + "logits/rejected": -2.1182327270507812, + "logps/chosen": -258.93572998046875, + "logps/rejected": -209.24758911132812, + "loss": 1536.6931, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.096510149538517, + "rewards/margins": 0.16783210635185242, + "rewards/rejected": -0.2643422484397888, + "step": 12460 + }, + { + "epoch": 0.82, + "learning_rate": 4.988102237982454e-07, + "logits/chosen": -2.390838146209717, + "logits/rejected": -2.3238790035247803, + "logps/chosen": -236.59536743164062, + "logps/rejected": -207.47622680664062, + "loss": 1904.8795, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1441490352153778, + "rewards/margins": 0.1145612969994545, + "rewards/rejected": -0.2587103247642517, + "step": 12470 + }, + { + "epoch": 0.82, + "learning_rate": 4.953931263846251e-07, + "logits/chosen": -2.3809030055999756, + "logits/rejected": -2.1044983863830566, + "logps/chosen": -272.463134765625, + "logps/rejected": -244.44815063476562, + "loss": 1726.9566, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.10754238069057465, + "rewards/margins": 0.1594572365283966, + "rewards/rejected": -0.26699960231781006, + "step": 12480 + }, + { + "epoch": 0.82, + "learning_rate": 4.919864859824266e-07, + "logits/chosen": -2.3234763145446777, + "logits/rejected": -2.1786303520202637, + "logps/chosen": -252.82870483398438, + "logps/rejected": -220.4989013671875, + "loss": 1900.4014, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15394526720046997, + "rewards/margins": 0.12705360352993011, + "rewards/rejected": -0.2809988856315613, + "step": 12490 + }, + { + "epoch": 0.82, + "learning_rate": 4.885903203623532e-07, + "logits/chosen": -2.498518466949463, + "logits/rejected": -2.107308864593506, + "logps/chosen": -290.13751220703125, + "logps/rejected": -249.6146697998047, + "loss": 1238.4372, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.051236413419246674, + "rewards/margins": 0.23250535130500793, + "rewards/rejected": -0.2837417721748352, + "step": 12500 + }, + { + "epoch": 0.82, + "eval_logits/chosen": -2.3528501987457275, + "eval_logits/rejected": -2.1651055812835693, + "eval_logps/chosen": -241.78085327148438, + "eval_logps/rejected": -236.79336547851562, + "eval_loss": 1760.1107177734375, + "eval_rewards/accuracies": 0.6610000133514404, + "eval_rewards/chosen": -0.09775892645120621, + "eval_rewards/margins": 0.15405558049678802, + "eval_rewards/rejected": -0.2518145442008972, + "eval_runtime": 725.376, + "eval_samples_per_second": 2.757, + "eval_steps_per_second": 1.379, + "step": 12500 + }, + { + "epoch": 0.82, + "learning_rate": 4.852046472404695e-07, + "logits/chosen": -2.499540328979492, + "logits/rejected": -1.7627403736114502, + "logps/chosen": -305.05047607421875, + "logps/rejected": -195.26817321777344, + "loss": 1706.6885, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.07161720097064972, + "rewards/margins": 0.1589113175868988, + "rewards/rejected": -0.23052850365638733, + "step": 12510 + }, + { + "epoch": 0.82, + "learning_rate": 4.818294842781035e-07, + "logits/chosen": -2.41566801071167, + "logits/rejected": -2.240298271179199, + "logps/chosen": -234.0768280029297, + "logps/rejected": -200.38133239746094, + "loss": 1719.5809, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.07868404686450958, + "rewards/margins": 0.18662366271018982, + "rewards/rejected": -0.2653077244758606, + "step": 12520 + }, + { + "epoch": 0.82, + "learning_rate": 4.784648490817601e-07, + "logits/chosen": -2.413585662841797, + "logits/rejected": -2.120288848876953, + "logps/chosen": -231.2106170654297, + "logps/rejected": -201.527587890625, + "loss": 1995.8398, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11103849112987518, + "rewards/margins": 0.13575485348701477, + "rewards/rejected": -0.24679334461688995, + "step": 12530 + }, + { + "epoch": 0.82, + "learning_rate": 4.751107592030235e-07, + "logits/chosen": -2.427438259124756, + "logits/rejected": -2.1439523696899414, + "logps/chosen": -177.9643096923828, + "logps/rejected": -182.3109588623047, + "loss": 1420.8544, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.0811251550912857, + "rewards/margins": 0.2144661396741867, + "rewards/rejected": -0.2955912947654724, + "step": 12540 + }, + { + "epoch": 0.82, + "learning_rate": 4.717672321384703e-07, + "logits/chosen": -2.3183751106262207, + "logits/rejected": -2.0838351249694824, + "logps/chosen": -229.7345428466797, + "logps/rejected": -203.35838317871094, + "loss": 1716.082, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.06488841772079468, + "rewards/margins": 0.1581958383321762, + "rewards/rejected": -0.22308428585529327, + "step": 12550 + }, + { + "epoch": 0.82, + "learning_rate": 4.684342853295748e-07, + "logits/chosen": -2.2812492847442627, + "logits/rejected": -2.1551685333251953, + "logps/chosen": -193.63571166992188, + "logps/rejected": -208.3316192626953, + "loss": 1547.6708, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0593893900513649, + "rewards/margins": 0.1861993670463562, + "rewards/rejected": -0.2455887496471405, + "step": 12560 + }, + { + "epoch": 0.82, + "learning_rate": 4.651119361626213e-07, + "logits/chosen": -2.557042360305786, + "logits/rejected": -2.228407144546509, + "logps/chosen": -248.6354522705078, + "logps/rejected": -216.62582397460938, + "loss": 1680.7773, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07516884058713913, + "rewards/margins": 0.14219233393669128, + "rewards/rejected": -0.21736116707324982, + "step": 12570 + }, + { + "epoch": 0.82, + "learning_rate": 4.618002019686091e-07, + "logits/chosen": -2.3262791633605957, + "logits/rejected": -2.1434736251831055, + "logps/chosen": -289.65655517578125, + "logps/rejected": -247.00265502929688, + "loss": 2110.5848, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10444200038909912, + "rewards/margins": 0.13982179760932922, + "rewards/rejected": -0.24426379799842834, + "step": 12580 + }, + { + "epoch": 0.82, + "learning_rate": 4.5849910002316757e-07, + "logits/chosen": -2.408716917037964, + "logits/rejected": -2.06657075881958, + "logps/chosen": -193.36878967285156, + "logps/rejected": -177.7703399658203, + "loss": 1534.195, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12856633961200714, + "rewards/margins": 0.17665782570838928, + "rewards/rejected": -0.30522421002388, + "step": 12590 + }, + { + "epoch": 0.82, + "learning_rate": 4.5520864754645984e-07, + "logits/chosen": -2.434593915939331, + "logits/rejected": -2.3042731285095215, + "logps/chosen": -289.6260070800781, + "logps/rejected": -257.13055419921875, + "loss": 1967.9301, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.0827619805932045, + "rewards/margins": 0.11329551786184311, + "rewards/rejected": -0.1960574835538864, + "step": 12600 + }, + { + "epoch": 0.82, + "eval_logits/chosen": -2.3529741764068604, + "eval_logits/rejected": -2.165254592895508, + "eval_logps/chosen": -241.44981384277344, + "eval_logps/rejected": -236.67112731933594, + "eval_loss": 1757.7244873046875, + "eval_rewards/accuracies": 0.6614999771118164, + "eval_rewards/chosen": -0.09444864839315414, + "eval_rewards/margins": 0.15614351630210876, + "eval_rewards/rejected": -0.2505921423435211, + "eval_runtime": 721.4429, + "eval_samples_per_second": 2.772, + "eval_steps_per_second": 1.386, + "step": 12600 + }, + { + "epoch": 0.83, + "learning_rate": 4.5192886170309896e-07, + "logits/chosen": -2.2827320098876953, + "logits/rejected": -2.179940700531006, + "logps/chosen": -201.1009521484375, + "logps/rejected": -213.9563446044922, + "loss": 2239.9746, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.10332496464252472, + "rewards/margins": 0.12671324610710144, + "rewards/rejected": -0.23003819584846497, + "step": 12610 + }, + { + "epoch": 0.83, + "learning_rate": 4.486597596020548e-07, + "logits/chosen": -2.3789055347442627, + "logits/rejected": -2.1024844646453857, + "logps/chosen": -231.01260375976562, + "logps/rejected": -211.09994506835938, + "loss": 1544.9164, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.11431293189525604, + "rewards/margins": 0.18409089744091034, + "rewards/rejected": -0.29840385913848877, + "step": 12620 + }, + { + "epoch": 0.83, + "learning_rate": 4.454013582965644e-07, + "logits/chosen": -2.304321527481079, + "logits/rejected": -1.8886350393295288, + "logps/chosen": -274.82427978515625, + "logps/rejected": -236.3057861328125, + "loss": 2003.2834, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11367567628622055, + "rewards/margins": 0.11924811452627182, + "rewards/rejected": -0.23292379081249237, + "step": 12630 + }, + { + "epoch": 0.83, + "learning_rate": 4.4215367478404605e-07, + "logits/chosen": -2.148632287979126, + "logits/rejected": -2.125662326812744, + "logps/chosen": -283.1051940917969, + "logps/rejected": -322.2588195800781, + "loss": 1788.9277, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1234527975320816, + "rewards/margins": 0.14781855046749115, + "rewards/rejected": -0.27127137780189514, + "step": 12640 + }, + { + "epoch": 0.83, + "learning_rate": 4.389167260060068e-07, + "logits/chosen": -2.4365358352661133, + "logits/rejected": -2.1948304176330566, + "logps/chosen": -213.0283660888672, + "logps/rejected": -195.78033447265625, + "loss": 1292.1158, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.04338112473487854, + "rewards/margins": 0.20926494896411896, + "rewards/rejected": -0.2526460886001587, + "step": 12650 + }, + { + "epoch": 0.83, + "learning_rate": 4.356905288479579e-07, + "logits/chosen": -2.2978675365448, + "logits/rejected": -2.0709261894226074, + "logps/chosen": -234.98574829101562, + "logps/rejected": -227.6422119140625, + "loss": 1213.8726, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.08464913815259933, + "rewards/margins": 0.2142043113708496, + "rewards/rejected": -0.29885345697402954, + "step": 12660 + }, + { + "epoch": 0.83, + "learning_rate": 4.3247510013932377e-07, + "logits/chosen": -2.248154640197754, + "logits/rejected": -2.078132152557373, + "logps/chosen": -261.9805908203125, + "logps/rejected": -279.871826171875, + "loss": 1742.7143, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09792111814022064, + "rewards/margins": 0.18071343004703522, + "rewards/rejected": -0.27863457798957825, + "step": 12670 + }, + { + "epoch": 0.83, + "learning_rate": 4.2927045665335594e-07, + "logits/chosen": -1.9597619771957397, + "logits/rejected": -1.8733116388320923, + "logps/chosen": -181.56686401367188, + "logps/rejected": -189.19589233398438, + "loss": 1980.1352, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1668912172317505, + "rewards/margins": 0.11184272915124893, + "rewards/rejected": -0.27873390913009644, + "step": 12680 + }, + { + "epoch": 0.83, + "learning_rate": 4.260766151070439e-07, + "logits/chosen": -2.1650466918945312, + "logits/rejected": -2.2159152030944824, + "logps/chosen": -236.6437530517578, + "logps/rejected": -241.3082733154297, + "loss": 1697.1186, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.12210245430469513, + "rewards/margins": 0.15925151109695435, + "rewards/rejected": -0.2813539505004883, + "step": 12690 + }, + { + "epoch": 0.83, + "learning_rate": 4.228935921610308e-07, + "logits/chosen": -2.3920722007751465, + "logits/rejected": -2.031923532485962, + "logps/chosen": -273.67059326171875, + "logps/rejected": -223.01467895507812, + "loss": 1912.1277, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09951961785554886, + "rewards/margins": 0.13395579159259796, + "rewards/rejected": -0.23347540199756622, + "step": 12700 + }, + { + "epoch": 0.83, + "eval_logits/chosen": -2.3546695709228516, + "eval_logits/rejected": -2.1666719913482666, + "eval_logps/chosen": -241.46438598632812, + "eval_logps/rejected": -236.64071655273438, + "eval_loss": 1757.38720703125, + "eval_rewards/accuracies": 0.6620000004768372, + "eval_rewards/chosen": -0.09459412097930908, + "eval_rewards/margins": 0.15569402277469635, + "eval_rewards/rejected": -0.25028812885284424, + "eval_runtime": 723.8008, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.382, + "step": 12700 + }, + { + "epoch": 0.83, + "learning_rate": 4.1972140441952246e-07, + "logits/chosen": -2.2060556411743164, + "logits/rejected": -2.172612428665161, + "logps/chosen": -247.1871795654297, + "logps/rejected": -265.71710205078125, + "loss": 1742.0557, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07603900879621506, + "rewards/margins": 0.1652071326971054, + "rewards/rejected": -0.24124614894390106, + "step": 12710 + }, + { + "epoch": 0.83, + "learning_rate": 4.165600684302046e-07, + "logits/chosen": -2.3115181922912598, + "logits/rejected": -2.372889518737793, + "logps/chosen": -182.8769989013672, + "logps/rejected": -209.275634765625, + "loss": 1456.5319, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0709301307797432, + "rewards/margins": 0.1644066870212555, + "rewards/rejected": -0.23533682525157928, + "step": 12720 + }, + { + "epoch": 0.83, + "learning_rate": 4.13409600684154e-07, + "logits/chosen": -2.4179084300994873, + "logits/rejected": -2.1326534748077393, + "logps/chosen": -226.34835815429688, + "logps/rejected": -209.9674835205078, + "loss": 1859.2658, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.11328748613595963, + "rewards/margins": 0.12802428007125854, + "rewards/rejected": -0.24131174385547638, + "step": 12730 + }, + { + "epoch": 0.83, + "learning_rate": 4.102700176157548e-07, + "logits/chosen": -2.4787747859954834, + "logits/rejected": -2.1119885444641113, + "logps/chosen": -335.3193664550781, + "logps/rejected": -254.7163543701172, + "loss": 1789.3066, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.10564114153385162, + "rewards/margins": 0.1718355268239975, + "rewards/rejected": -0.2774766683578491, + "step": 12740 + }, + { + "epoch": 0.83, + "learning_rate": 4.0714133560260884e-07, + "logits/chosen": -2.366929292678833, + "logits/rejected": -2.200700283050537, + "logps/chosen": -272.04559326171875, + "logps/rejected": -226.88577270507812, + "loss": 1714.9975, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12512561678886414, + "rewards/margins": 0.13071008026599884, + "rewards/rejected": -0.25583571195602417, + "step": 12750 + }, + { + "epoch": 0.83, + "learning_rate": 4.0402357096545527e-07, + "logits/chosen": -2.249117374420166, + "logits/rejected": -2.2036967277526855, + "logps/chosen": -259.86865234375, + "logps/rejected": -264.2768249511719, + "loss": 1702.4238, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08107040822505951, + "rewards/margins": 0.1458026021718979, + "rewards/rejected": -0.2268730103969574, + "step": 12760 + }, + { + "epoch": 0.84, + "learning_rate": 4.0091673996808025e-07, + "logits/chosen": -2.456432342529297, + "logits/rejected": -2.2576537132263184, + "logps/chosen": -212.8669891357422, + "logps/rejected": -201.2241973876953, + "loss": 1807.799, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.15806898474693298, + "rewards/margins": 0.12984392046928406, + "rewards/rejected": -0.28791293501853943, + "step": 12770 + }, + { + "epoch": 0.84, + "learning_rate": 3.9782085881723776e-07, + "logits/chosen": -2.279867649078369, + "logits/rejected": -2.130587339401245, + "logps/chosen": -177.62420654296875, + "logps/rejected": -204.5876007080078, + "loss": 1886.7762, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.12026622146368027, + "rewards/margins": 0.14996710419654846, + "rewards/rejected": -0.2702333331108093, + "step": 12780 + }, + { + "epoch": 0.84, + "learning_rate": 3.947359436625592e-07, + "logits/chosen": -2.2889564037323, + "logits/rejected": -2.1877830028533936, + "logps/chosen": -246.07382202148438, + "logps/rejected": -225.522705078125, + "loss": 1585.2244, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11410894244909286, + "rewards/margins": 0.17013028264045715, + "rewards/rejected": -0.284239262342453, + "step": 12790 + }, + { + "epoch": 0.84, + "learning_rate": 3.9166201059647386e-07, + "logits/chosen": -2.4249634742736816, + "logits/rejected": -2.2888290882110596, + "logps/chosen": -273.11492919921875, + "logps/rejected": -243.9048614501953, + "loss": 2416.5143, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.11264701187610626, + "rewards/margins": 0.07720918953418732, + "rewards/rejected": -0.18985618650913239, + "step": 12800 + }, + { + "epoch": 0.84, + "eval_logits/chosen": -2.351189136505127, + "eval_logits/rejected": -2.163454055786133, + "eval_logps/chosen": -242.81185913085938, + "eval_logps/rejected": -237.82569885253906, + "eval_loss": 1756.477294921875, + "eval_rewards/accuracies": 0.6610000133514404, + "eval_rewards/chosen": -0.10806912183761597, + "eval_rewards/margins": 0.15406881272792816, + "eval_rewards/rejected": -0.2621379494667053, + "eval_runtime": 724.1571, + "eval_samples_per_second": 2.762, + "eval_steps_per_second": 1.381, + "step": 12800 + }, + { + "epoch": 0.84, + "learning_rate": 3.8859907565412194e-07, + "logits/chosen": -2.2357773780822754, + "logits/rejected": -2.3141744136810303, + "logps/chosen": -193.81643676757812, + "logps/rejected": -209.11569213867188, + "loss": 1652.4705, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14167191088199615, + "rewards/margins": 0.1587766706943512, + "rewards/rejected": -0.30044859647750854, + "step": 12810 + }, + { + "epoch": 0.84, + "learning_rate": 3.8554715481327303e-07, + "logits/chosen": -2.3908543586730957, + "logits/rejected": -1.998772382736206, + "logps/chosen": -247.39334106445312, + "logps/rejected": -225.96493530273438, + "loss": 2023.8736, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.14134298264980316, + "rewards/margins": 0.1396934986114502, + "rewards/rejected": -0.28103652596473694, + "step": 12820 + }, + { + "epoch": 0.84, + "learning_rate": 3.8250626399424007e-07, + "logits/chosen": -2.4040729999542236, + "logits/rejected": -2.1485342979431152, + "logps/chosen": -264.1070251464844, + "logps/rejected": -259.52252197265625, + "loss": 1815.5469, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12628862261772156, + "rewards/margins": 0.14265701174736023, + "rewards/rejected": -0.2689456343650818, + "step": 12830 + }, + { + "epoch": 0.84, + "learning_rate": 3.7947641905980104e-07, + "logits/chosen": -2.229609251022339, + "logits/rejected": -2.2407336235046387, + "logps/chosen": -215.80990600585938, + "logps/rejected": -196.70230102539062, + "loss": 1642.165, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08491475880146027, + "rewards/margins": 0.15573066473007202, + "rewards/rejected": -0.24064543843269348, + "step": 12840 + }, + { + "epoch": 0.84, + "learning_rate": 3.764576358151098e-07, + "logits/chosen": -2.263810634613037, + "logits/rejected": -2.2646942138671875, + "logps/chosen": -186.5194091796875, + "logps/rejected": -185.54440307617188, + "loss": 1978.7371, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11702384799718857, + "rewards/margins": 0.10869280993938446, + "rewards/rejected": -0.22571666538715363, + "step": 12850 + }, + { + "epoch": 0.84, + "learning_rate": 3.7344993000761944e-07, + "logits/chosen": -2.376746654510498, + "logits/rejected": -2.2694809436798096, + "logps/chosen": -191.15414428710938, + "logps/rejected": -249.1569061279297, + "loss": 1602.7209, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13839024305343628, + "rewards/margins": 0.15030980110168457, + "rewards/rejected": -0.28870004415512085, + "step": 12860 + }, + { + "epoch": 0.84, + "learning_rate": 3.7045331732699585e-07, + "logits/chosen": -2.380181312561035, + "logits/rejected": -2.1943130493164062, + "logps/chosen": -216.8921356201172, + "logps/rejected": -199.8214874267578, + "loss": 1621.9516, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11694061756134033, + "rewards/margins": 0.18628790974617004, + "rewards/rejected": -0.30322855710983276, + "step": 12870 + }, + { + "epoch": 0.84, + "learning_rate": 3.6746781340503993e-07, + "logits/chosen": -2.2263026237487793, + "logits/rejected": -2.1297569274902344, + "logps/chosen": -243.273681640625, + "logps/rejected": -248.5195770263672, + "loss": 1795.5994, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08857928216457367, + "rewards/margins": 0.13272887468338013, + "rewards/rejected": -0.2213081419467926, + "step": 12880 + }, + { + "epoch": 0.84, + "learning_rate": 3.6449343381560116e-07, + "logits/chosen": -2.3246045112609863, + "logits/rejected": -2.105888843536377, + "logps/chosen": -247.8300323486328, + "logps/rejected": -250.9560089111328, + "loss": 1695.3215, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16404370963573456, + "rewards/margins": 0.1651550531387329, + "rewards/rejected": -0.32919877767562866, + "step": 12890 + }, + { + "epoch": 0.84, + "learning_rate": 3.615301940745017e-07, + "logits/chosen": -2.5765836238861084, + "logits/rejected": -1.9856640100479126, + "logps/chosen": -325.85577392578125, + "logps/rejected": -236.7607421875, + "loss": 1621.5686, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08660654723644257, + "rewards/margins": 0.16081905364990234, + "rewards/rejected": -0.2474256008863449, + "step": 12900 + }, + { + "epoch": 0.84, + "eval_logits/chosen": -2.3496968746185303, + "eval_logits/rejected": -2.1621196269989014, + "eval_logps/chosen": -243.3934783935547, + "eval_logps/rejected": -238.40719604492188, + "eval_loss": 1755.8291015625, + "eval_rewards/accuracies": 0.6650000214576721, + "eval_rewards/chosen": -0.1138853132724762, + "eval_rewards/margins": 0.1540677845478058, + "eval_rewards/rejected": -0.267953097820282, + "eval_runtime": 723.5217, + "eval_samples_per_second": 2.764, + "eval_steps_per_second": 1.382, + "step": 12900 + }, + { + "epoch": 0.84, + "learning_rate": 3.5857810963945084e-07, + "logits/chosen": -2.218391180038452, + "logits/rejected": -1.9942808151245117, + "logps/chosen": -227.6157989501953, + "logps/rejected": -229.28500366210938, + "loss": 1633.0026, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1310669481754303, + "rewards/margins": 0.16492125391960144, + "rewards/rejected": -0.29598820209503174, + "step": 12910 + }, + { + "epoch": 0.85, + "learning_rate": 3.556371959099678e-07, + "logits/chosen": -2.4070942401885986, + "logits/rejected": -2.1696436405181885, + "logps/chosen": -308.32635498046875, + "logps/rejected": -283.7373962402344, + "loss": 1785.7984, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11152037233114243, + "rewards/margins": 0.15381523966789246, + "rewards/rejected": -0.2653356194496155, + "step": 12920 + }, + { + "epoch": 0.85, + "learning_rate": 3.5270746822729797e-07, + "logits/chosen": -2.294844388961792, + "logits/rejected": -2.2179951667785645, + "logps/chosen": -263.7327880859375, + "logps/rejected": -288.92022705078125, + "loss": 2077.7789, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1625073105096817, + "rewards/margins": 0.12612196803092957, + "rewards/rejected": -0.2886292636394501, + "step": 12930 + }, + { + "epoch": 0.85, + "learning_rate": 3.4978894187433746e-07, + "logits/chosen": -2.4065170288085938, + "logits/rejected": -2.266335964202881, + "logps/chosen": -165.07269287109375, + "logps/rejected": -163.32843017578125, + "loss": 2126.6654, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13997486233711243, + "rewards/margins": 0.09164164960384369, + "rewards/rejected": -0.2316165268421173, + "step": 12940 + }, + { + "epoch": 0.85, + "learning_rate": 3.468816320755486e-07, + "logits/chosen": -2.1857643127441406, + "logits/rejected": -2.0050368309020996, + "logps/chosen": -233.8216552734375, + "logps/rejected": -206.8753662109375, + "loss": 1907.748, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09893886744976044, + "rewards/margins": 0.12663452327251434, + "rewards/rejected": -0.2255733758211136, + "step": 12950 + }, + { + "epoch": 0.85, + "learning_rate": 3.4398555399688336e-07, + "logits/chosen": -2.4522571563720703, + "logits/rejected": -2.0734975337982178, + "logps/chosen": -228.2733612060547, + "logps/rejected": -213.73123168945312, + "loss": 2365.5295, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1692286729812622, + "rewards/margins": 0.06122065708041191, + "rewards/rejected": -0.2304493486881256, + "step": 12960 + }, + { + "epoch": 0.85, + "learning_rate": 3.411007227457047e-07, + "logits/chosen": -2.3396944999694824, + "logits/rejected": -2.292318820953369, + "logps/chosen": -257.23907470703125, + "logps/rejected": -240.35848999023438, + "loss": 1584.4968, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09477768838405609, + "rewards/margins": 0.1539420336484909, + "rewards/rejected": -0.248719722032547, + "step": 12970 + }, + { + "epoch": 0.85, + "learning_rate": 3.382271533707043e-07, + "logits/chosen": -2.2677955627441406, + "logits/rejected": -2.254593849182129, + "logps/chosen": -201.43359375, + "logps/rejected": -191.1988525390625, + "loss": 1955.8822, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0982837826013565, + "rewards/margins": 0.12648960947990417, + "rewards/rejected": -0.2247733771800995, + "step": 12980 + }, + { + "epoch": 0.85, + "learning_rate": 3.353648608618287e-07, + "logits/chosen": -2.341491937637329, + "logits/rejected": -2.0881080627441406, + "logps/chosen": -180.15113830566406, + "logps/rejected": -183.06163024902344, + "loss": 1867.4459, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12279865890741348, + "rewards/margins": 0.12881214916706085, + "rewards/rejected": -0.25161081552505493, + "step": 12990 + }, + { + "epoch": 0.85, + "learning_rate": 3.3251386015019676e-07, + "logits/chosen": -2.368988513946533, + "logits/rejected": -2.157670736312866, + "logps/chosen": -214.96731567382812, + "logps/rejected": -194.64791870117188, + "loss": 1645.2689, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1220787763595581, + "rewards/margins": 0.13425137102603912, + "rewards/rejected": -0.25633013248443604, + "step": 13000 + }, + { + "epoch": 0.85, + "eval_logits/chosen": -2.3476548194885254, + "eval_logits/rejected": -2.1602706909179688, + "eval_logps/chosen": -243.2641143798828, + "eval_logps/rejected": -238.19064331054688, + "eval_loss": 1755.0059814453125, + "eval_rewards/accuracies": 0.6639999747276306, + "eval_rewards/chosen": -0.11259139329195023, + "eval_rewards/margins": 0.15319600701332092, + "eval_rewards/rejected": -0.26578739285469055, + "eval_runtime": 724.8273, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 13000 + }, + { + "epoch": 0.85, + "learning_rate": 3.296741661080255e-07, + "logits/chosen": -2.2990310192108154, + "logits/rejected": -2.212502956390381, + "logps/chosen": -249.18881225585938, + "logps/rejected": -255.80764770507812, + "loss": 2351.9094, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1605045646429062, + "rewards/margins": 0.13160006701946259, + "rewards/rejected": -0.2921046316623688, + "step": 13010 + }, + { + "epoch": 0.85, + "learning_rate": 3.2684579354854974e-07, + "logits/chosen": -2.391723394393921, + "logits/rejected": -2.286519765853882, + "logps/chosen": -306.3222351074219, + "logps/rejected": -334.36419677734375, + "loss": 1797.0959, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13863664865493774, + "rewards/margins": 0.17783468961715698, + "rewards/rejected": -0.3164713382720947, + "step": 13020 + }, + { + "epoch": 0.85, + "learning_rate": 3.2402875722594653e-07, + "logits/chosen": -2.4300198554992676, + "logits/rejected": -2.1459197998046875, + "logps/chosen": -179.18894958496094, + "logps/rejected": -200.12417602539062, + "loss": 1694.7115, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10143520683050156, + "rewards/margins": 0.14783641695976257, + "rewards/rejected": -0.24927166104316711, + "step": 13030 + }, + { + "epoch": 0.85, + "learning_rate": 3.212230718352566e-07, + "logits/chosen": -2.2786428928375244, + "logits/rejected": -2.298488140106201, + "logps/chosen": -238.57382202148438, + "logps/rejected": -179.4725341796875, + "loss": 2324.5529, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13523949682712555, + "rewards/margins": 0.04595714807510376, + "rewards/rejected": -0.1811966449022293, + "step": 13040 + }, + { + "epoch": 0.85, + "learning_rate": 3.1842875201231025e-07, + "logits/chosen": -2.3447372913360596, + "logits/rejected": -2.0560357570648193, + "logps/chosen": -234.4106903076172, + "logps/rejected": -222.1627655029297, + "loss": 1798.5451, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12670625746250153, + "rewards/margins": 0.1352727711200714, + "rewards/rejected": -0.26197901368141174, + "step": 13050 + }, + { + "epoch": 0.85, + "learning_rate": 3.156458123336478e-07, + "logits/chosen": -2.189135789871216, + "logits/rejected": -2.0165061950683594, + "logps/chosen": -169.20169067382812, + "logps/rejected": -177.94007873535156, + "loss": 1615.8548, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.10689838975667953, + "rewards/margins": 0.18887147307395935, + "rewards/rejected": -0.2957698702812195, + "step": 13060 + }, + { + "epoch": 0.86, + "learning_rate": 3.128742673164459e-07, + "logits/chosen": -2.415038585662842, + "logits/rejected": -2.054231643676758, + "logps/chosen": -292.4279479980469, + "logps/rejected": -268.6998596191406, + "loss": 1283.446, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.09500737488269806, + "rewards/margins": 0.20787818729877472, + "rewards/rejected": -0.3028855621814728, + "step": 13070 + }, + { + "epoch": 0.86, + "learning_rate": 3.101141314184414e-07, + "logits/chosen": -2.525285005569458, + "logits/rejected": -2.282944679260254, + "logps/chosen": -213.61471557617188, + "logps/rejected": -220.43521118164062, + "loss": 1461.5715, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.0823017954826355, + "rewards/margins": 0.1603173315525055, + "rewards/rejected": -0.242619127035141, + "step": 13080 + }, + { + "epoch": 0.86, + "learning_rate": 3.0736541903785526e-07, + "logits/chosen": -2.1892294883728027, + "logits/rejected": -2.151439666748047, + "logps/chosen": -223.6445770263672, + "logps/rejected": -286.0027160644531, + "loss": 1937.7354, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1462966501712799, + "rewards/margins": 0.1289856731891632, + "rewards/rejected": -0.2752823233604431, + "step": 13090 + }, + { + "epoch": 0.86, + "learning_rate": 3.0462814451331704e-07, + "logits/chosen": -2.23403000831604, + "logits/rejected": -2.103334903717041, + "logps/chosen": -246.27597045898438, + "logps/rejected": -254.24423217773438, + "loss": 2114.2795, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.15599580109119415, + "rewards/margins": 0.08063185960054398, + "rewards/rejected": -0.23662766814231873, + "step": 13100 + }, + { + "epoch": 0.86, + "eval_logits/chosen": -2.345999240875244, + "eval_logits/rejected": -2.158658266067505, + "eval_logps/chosen": -243.78396606445312, + "eval_logps/rejected": -238.8558807373047, + "eval_loss": 1753.9947509765625, + "eval_rewards/accuracies": 0.6650000214576721, + "eval_rewards/chosen": -0.11779005825519562, + "eval_rewards/margins": 0.1546495258808136, + "eval_rewards/rejected": -0.272439569234848, + "eval_runtime": 724.8177, + "eval_samples_per_second": 2.759, + "eval_steps_per_second": 1.38, + "step": 13100 + }, + { + "epoch": 0.86, + "learning_rate": 3.019023221237927e-07, + "logits/chosen": -2.2977519035339355, + "logits/rejected": -2.1277058124542236, + "logps/chosen": -248.1866912841797, + "logps/rejected": -207.8753204345703, + "loss": 1654.5158, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11454341560602188, + "rewards/margins": 0.17256048321723938, + "rewards/rejected": -0.28710389137268066, + "step": 13110 + }, + { + "epoch": 0.86, + "learning_rate": 2.991879660885058e-07, + "logits/chosen": -2.46199893951416, + "logits/rejected": -2.2021241188049316, + "logps/chosen": -276.08203125, + "logps/rejected": -275.280029296875, + "loss": 1771.293, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1253511607646942, + "rewards/margins": 0.15558239817619324, + "rewards/rejected": -0.28093355894088745, + "step": 13120 + }, + { + "epoch": 0.86, + "learning_rate": 2.9648509056686786e-07, + "logits/chosen": -2.3713479042053223, + "logits/rejected": -2.2038354873657227, + "logps/chosen": -187.05670166015625, + "logps/rejected": -175.7555389404297, + "loss": 1879.9361, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10114918649196625, + "rewards/margins": 0.14274568855762482, + "rewards/rejected": -0.24389486014842987, + "step": 13130 + }, + { + "epoch": 0.86, + "learning_rate": 2.937937096584012e-07, + "logits/chosen": -2.3491787910461426, + "logits/rejected": -2.131972551345825, + "logps/chosen": -302.1286926269531, + "logps/rejected": -256.2218017578125, + "loss": 1869.2926, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11727768182754517, + "rewards/margins": 0.1457705795764923, + "rewards/rejected": -0.2630482614040375, + "step": 13140 + }, + { + "epoch": 0.86, + "learning_rate": 2.9111383740266756e-07, + "logits/chosen": -2.1542248725891113, + "logits/rejected": -2.0185928344726562, + "logps/chosen": -249.68789672851562, + "logps/rejected": -256.65728759765625, + "loss": 1765.4994, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13903729617595673, + "rewards/margins": 0.12757453322410583, + "rewards/rejected": -0.26661184430122375, + "step": 13150 + }, + { + "epoch": 0.86, + "learning_rate": 2.8844548777919255e-07, + "logits/chosen": -2.395974636077881, + "logits/rejected": -2.113468647003174, + "logps/chosen": -213.87539672851562, + "logps/rejected": -203.4140167236328, + "loss": 2126.2518, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.09991560131311417, + "rewards/margins": 0.11818840354681015, + "rewards/rejected": -0.2181040346622467, + "step": 13160 + }, + { + "epoch": 0.86, + "learning_rate": 2.8578867470739594e-07, + "logits/chosen": -2.2456743717193604, + "logits/rejected": -2.1050240993499756, + "logps/chosen": -196.46435546875, + "logps/rejected": -183.75253295898438, + "loss": 1607.4187, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11976015567779541, + "rewards/margins": 0.18710866570472717, + "rewards/rejected": -0.3068688213825226, + "step": 13170 + }, + { + "epoch": 0.86, + "learning_rate": 2.8314341204651484e-07, + "logits/chosen": -2.450092315673828, + "logits/rejected": -2.181081771850586, + "logps/chosen": -281.95928955078125, + "logps/rejected": -228.1092529296875, + "loss": 1398.7136, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.05920163542032242, + "rewards/margins": 0.18932537734508514, + "rewards/rejected": -0.24852702021598816, + "step": 13180 + }, + { + "epoch": 0.86, + "learning_rate": 2.805097135955362e-07, + "logits/chosen": -2.3811404705047607, + "logits/rejected": -2.1690518856048584, + "logps/chosen": -223.0667724609375, + "logps/rejected": -202.48733520507812, + "loss": 1702.7156, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10542796552181244, + "rewards/margins": 0.1434640884399414, + "rewards/rejected": -0.24889206886291504, + "step": 13190 + }, + { + "epoch": 0.86, + "learning_rate": 2.778875930931213e-07, + "logits/chosen": -2.3781535625457764, + "logits/rejected": -2.0523698329925537, + "logps/chosen": -246.26998901367188, + "logps/rejected": -243.38485717773438, + "loss": 1719.5012, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11151113361120224, + "rewards/margins": 0.14477068185806274, + "rewards/rejected": -0.2562818229198456, + "step": 13200 + }, + { + "epoch": 0.86, + "eval_logits/chosen": -2.348130226135254, + "eval_logits/rejected": -2.160688638687134, + "eval_logps/chosen": -242.72756958007812, + "eval_logps/rejected": -238.03858947753906, + "eval_loss": 1755.7774658203125, + "eval_rewards/accuracies": 0.6629999876022339, + "eval_rewards/chosen": -0.10722615569829941, + "eval_rewards/margins": 0.15704070031642914, + "eval_rewards/rejected": -0.26426684856414795, + "eval_runtime": 724.6812, + "eval_samples_per_second": 2.76, + "eval_steps_per_second": 1.38, + "step": 13200 + }, + { + "epoch": 0.86, + "learning_rate": 2.7527706421753426e-07, + "logits/chosen": -2.3434062004089355, + "logits/rejected": -2.2829527854919434, + "logps/chosen": -209.6172637939453, + "logps/rejected": -225.86276245117188, + "loss": 1966.9658, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11216064542531967, + "rewards/margins": 0.11756797134876251, + "rewards/rejected": -0.2297286093235016, + "step": 13210 + }, + { + "epoch": 0.86, + "learning_rate": 2.726781405865736e-07, + "logits/chosen": -2.432314395904541, + "logits/rejected": -1.8859539031982422, + "logps/chosen": -313.43511962890625, + "logps/rejected": -212.71896362304688, + "loss": 1765.0092, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10769345611333847, + "rewards/margins": 0.150523841381073, + "rewards/rejected": -0.2582172751426697, + "step": 13220 + }, + { + "epoch": 0.87, + "learning_rate": 2.7009083575749687e-07, + "logits/chosen": -2.328192710876465, + "logits/rejected": -2.2318806648254395, + "logps/chosen": -256.3060607910156, + "logps/rejected": -267.0457458496094, + "loss": 1947.6164, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12445763498544693, + "rewards/margins": 0.12375207990407944, + "rewards/rejected": -0.24820971488952637, + "step": 13230 + }, + { + "epoch": 0.87, + "learning_rate": 2.6751516322695457e-07, + "logits/chosen": -2.405379056930542, + "logits/rejected": -2.33868408203125, + "logps/chosen": -202.52243041992188, + "logps/rejected": -205.9892578125, + "loss": 1987.4588, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12278878688812256, + "rewards/margins": 0.09506727755069733, + "rewards/rejected": -0.2178560495376587, + "step": 13240 + }, + { + "epoch": 0.87, + "learning_rate": 2.649511364309154e-07, + "logits/chosen": -2.3319852352142334, + "logits/rejected": -2.311432361602783, + "logps/chosen": -212.4677734375, + "logps/rejected": -207.6529083251953, + "loss": 1547.3972, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1017812117934227, + "rewards/margins": 0.16066250205039978, + "rewards/rejected": -0.2624437212944031, + "step": 13250 + }, + { + "epoch": 0.87, + "learning_rate": 2.6239876874460003e-07, + "logits/chosen": -2.439059019088745, + "logits/rejected": -2.315342426300049, + "logps/chosen": -294.7291564941406, + "logps/rejected": -285.35638427734375, + "loss": 1560.8948, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10313956439495087, + "rewards/margins": 0.2118341475725174, + "rewards/rejected": -0.31497371196746826, + "step": 13260 + }, + { + "epoch": 0.87, + "learning_rate": 2.5985807348240744e-07, + "logits/chosen": -2.495514154434204, + "logits/rejected": -2.031292200088501, + "logps/chosen": -238.3489227294922, + "logps/rejected": -214.0382080078125, + "loss": 1343.8792, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06005353853106499, + "rewards/margins": 0.23636405169963837, + "rewards/rejected": -0.29641759395599365, + "step": 13270 + }, + { + "epoch": 0.87, + "learning_rate": 2.5732906389785014e-07, + "logits/chosen": -2.366839647293091, + "logits/rejected": -2.2016913890838623, + "logps/chosen": -284.45574951171875, + "logps/rejected": -264.6962890625, + "loss": 1612.3196, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.06749661266803741, + "rewards/margins": 0.18386101722717285, + "rewards/rejected": -0.2513576149940491, + "step": 13280 + }, + { + "epoch": 0.87, + "learning_rate": 2.5481175318347956e-07, + "logits/chosen": -2.239269495010376, + "logits/rejected": -2.2813994884490967, + "logps/chosen": -235.7333221435547, + "logps/rejected": -270.91497802734375, + "loss": 1568.9496, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.08235698938369751, + "rewards/margins": 0.1728564202785492, + "rewards/rejected": -0.2552134096622467, + "step": 13290 + }, + { + "epoch": 0.87, + "learning_rate": 2.5230615447082246e-07, + "logits/chosen": -2.3373067378997803, + "logits/rejected": -1.9933191537857056, + "logps/chosen": -263.7798767089844, + "logps/rejected": -264.7540283203125, + "loss": 2001.4379, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12463216483592987, + "rewards/margins": 0.15759319067001343, + "rewards/rejected": -0.2822253704071045, + "step": 13300 + }, + { + "epoch": 0.87, + "eval_logits/chosen": -2.3474156856536865, + "eval_logits/rejected": -2.1600124835968018, + "eval_logps/chosen": -242.93479919433594, + "eval_logps/rejected": -238.28860473632812, + "eval_loss": 1754.6165771484375, + "eval_rewards/accuracies": 0.6635000109672546, + "eval_rewards/chosen": -0.10929837822914124, + "eval_rewards/margins": 0.15746866166591644, + "eval_rewards/rejected": -0.26676705479621887, + "eval_runtime": 724.4901, + "eval_samples_per_second": 2.761, + "eval_steps_per_second": 1.38, + "step": 13300 + }, + { + "epoch": 0.87, + "learning_rate": 2.49812280830308e-07, + "logits/chosen": -2.377274513244629, + "logits/rejected": -1.9086488485336304, + "logps/chosen": -236.3880157470703, + "logps/rejected": -226.5410614013672, + "loss": 1348.1293, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.11350829899311066, + "rewards/margins": 0.25009047985076904, + "rewards/rejected": -0.3635987639427185, + "step": 13310 + }, + { + "epoch": 0.87, + "learning_rate": 2.4733014527120457e-07, + "logits/chosen": -2.2500624656677246, + "logits/rejected": -2.0748238563537598, + "logps/chosen": -222.2921905517578, + "logps/rejected": -216.4639434814453, + "loss": 1680.1018, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19877322018146515, + "rewards/margins": 0.17064984142780304, + "rewards/rejected": -0.3694230318069458, + "step": 13320 + }, + { + "epoch": 0.87, + "learning_rate": 2.4485976074154565e-07, + "logits/chosen": -2.304439067840576, + "logits/rejected": -2.361459970474243, + "logps/chosen": -223.77297973632812, + "logps/rejected": -257.6127014160156, + "loss": 2405.968, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.1477496325969696, + "rewards/margins": 0.03796560689806938, + "rewards/rejected": -0.1857152134180069, + "step": 13330 + }, + { + "epoch": 0.87, + "learning_rate": 2.4240114012806763e-07, + "logits/chosen": -2.3313541412353516, + "logits/rejected": -2.3096864223480225, + "logps/chosen": -217.3317413330078, + "logps/rejected": -213.3277587890625, + "loss": 1960.4521, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09533555805683136, + "rewards/margins": 0.13065870106220245, + "rewards/rejected": -0.2259942591190338, + "step": 13340 + }, + { + "epoch": 0.87, + "learning_rate": 2.399542962561399e-07, + "logits/chosen": -2.2424967288970947, + "logits/rejected": -2.080265522003174, + "logps/chosen": -234.34274291992188, + "logps/rejected": -202.61734008789062, + "loss": 1645.757, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08886785805225372, + "rewards/margins": 0.1568392813205719, + "rewards/rejected": -0.24570715427398682, + "step": 13350 + }, + { + "epoch": 0.87, + "learning_rate": 2.3751924188969876e-07, + "logits/chosen": -2.2778706550598145, + "logits/rejected": -2.150668144226074, + "logps/chosen": -260.43463134765625, + "logps/rejected": -258.49066162109375, + "loss": 1610.7408, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.0869608074426651, + "rewards/margins": 0.1789068877696991, + "rewards/rejected": -0.265867680311203, + "step": 13360 + }, + { + "epoch": 0.87, + "learning_rate": 2.3509598973118024e-07, + "logits/chosen": -2.466491222381592, + "logits/rejected": -2.2724196910858154, + "logps/chosen": -233.4585418701172, + "logps/rejected": -181.59771728515625, + "loss": 2002.6088, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12453192472457886, + "rewards/margins": 0.10926959663629532, + "rewards/rejected": -0.23380151391029358, + "step": 13370 + }, + { + "epoch": 0.88, + "learning_rate": 2.326845524214555e-07, + "logits/chosen": -2.1327478885650635, + "logits/rejected": -2.2172887325286865, + "logps/chosen": -249.34310913085938, + "logps/rejected": -225.1226043701172, + "loss": 2794.0449, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1428876519203186, + "rewards/margins": 0.019042372703552246, + "rewards/rejected": -0.16193000972270966, + "step": 13380 + }, + { + "epoch": 0.88, + "learning_rate": 2.3028494253976158e-07, + "logits/chosen": -2.40440034866333, + "logits/rejected": -2.198272705078125, + "logps/chosen": -355.88104248046875, + "logps/rejected": -301.50421142578125, + "loss": 2115.4121, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14615550637245178, + "rewards/margins": 0.10204567015171051, + "rewards/rejected": -0.2482011765241623, + "step": 13390 + }, + { + "epoch": 0.88, + "learning_rate": 2.2789717260364026e-07, + "logits/chosen": -2.3983378410339355, + "logits/rejected": -2.2215027809143066, + "logps/chosen": -175.76011657714844, + "logps/rejected": -167.30177307128906, + "loss": 1796.4686, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.10654046386480331, + "rewards/margins": 0.13269367814064026, + "rewards/rejected": -0.23923416435718536, + "step": 13400 + }, + { + "epoch": 0.88, + "eval_logits/chosen": -2.3467912673950195, + "eval_logits/rejected": -2.1594457626342773, + "eval_logps/chosen": -243.21743774414062, + "eval_logps/rejected": -238.4784393310547, + "eval_loss": 1754.6705322265625, + "eval_rewards/accuracies": 0.6660000085830688, + "eval_rewards/chosen": -0.11212486773729324, + "eval_rewards/margins": 0.15654033422470093, + "eval_rewards/rejected": -0.2686651647090912, + "eval_runtime": 726.9658, + "eval_samples_per_second": 2.751, + "eval_steps_per_second": 1.376, + "step": 13400 + }, + { + "epoch": 0.88, + "learning_rate": 2.255212550688682e-07, + "logits/chosen": -2.296682357788086, + "logits/rejected": -2.3946003913879395, + "logps/chosen": -231.0108184814453, + "logps/rejected": -314.93560791015625, + "loss": 1660.6289, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.13055798411369324, + "rewards/margins": 0.19738134741783142, + "rewards/rejected": -0.32793933153152466, + "step": 13410 + }, + { + "epoch": 0.88, + "learning_rate": 2.2315720232939598e-07, + "logits/chosen": -2.6522374153137207, + "logits/rejected": -2.199693441390991, + "logps/chosen": -269.6587829589844, + "logps/rejected": -201.1451873779297, + "loss": 1347.6682, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07545066624879837, + "rewards/margins": 0.21088755130767822, + "rewards/rejected": -0.286338210105896, + "step": 13420 + }, + { + "epoch": 0.88, + "learning_rate": 2.2080502671727956e-07, + "logits/chosen": -2.4427454471588135, + "logits/rejected": -2.0991287231445312, + "logps/chosen": -229.20632934570312, + "logps/rejected": -221.53076171875, + "loss": 1802.2229, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.09124470502138138, + "rewards/margins": 0.14152109622955322, + "rewards/rejected": -0.2327658236026764, + "step": 13430 + }, + { + "epoch": 0.88, + "learning_rate": 2.1846474050262078e-07, + "logits/chosen": -2.417556047439575, + "logits/rejected": -2.2897114753723145, + "logps/chosen": -256.66497802734375, + "logps/rejected": -205.99191284179688, + "loss": 1681.0145, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10104218870401382, + "rewards/margins": 0.15124091506004333, + "rewards/rejected": -0.25228309631347656, + "step": 13440 + }, + { + "epoch": 0.88, + "learning_rate": 2.1613635589349756e-07, + "logits/chosen": -1.9890124797821045, + "logits/rejected": -2.082338809967041, + "logps/chosen": -205.3116455078125, + "logps/rejected": -252.1767120361328, + "loss": 1561.8028, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.08148379623889923, + "rewards/margins": 0.19344353675842285, + "rewards/rejected": -0.2749273180961609, + "step": 13450 + }, + { + "epoch": 0.88, + "learning_rate": 2.1381988503590578e-07, + "logits/chosen": -2.0807101726531982, + "logits/rejected": -2.1482956409454346, + "logps/chosen": -226.9981689453125, + "logps/rejected": -239.90011596679688, + "loss": 1403.4047, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.11126313358545303, + "rewards/margins": 0.19051769375801086, + "rewards/rejected": -0.3017807900905609, + "step": 13460 + }, + { + "epoch": 0.88, + "learning_rate": 2.11515340013691e-07, + "logits/chosen": -2.4283270835876465, + "logits/rejected": -2.40568208694458, + "logps/chosen": -241.0494384765625, + "logps/rejected": -240.2838134765625, + "loss": 1386.8065, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.09876888245344162, + "rewards/margins": 0.2081459015607834, + "rewards/rejected": -0.3069148063659668, + "step": 13470 + }, + { + "epoch": 0.88, + "learning_rate": 2.092227328484897e-07, + "logits/chosen": -2.212730646133423, + "logits/rejected": -2.1704113483428955, + "logps/chosen": -215.0039520263672, + "logps/rejected": -262.45782470703125, + "loss": 1706.5404, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10995247215032578, + "rewards/margins": 0.1679636687040329, + "rewards/rejected": -0.2779161334037781, + "step": 13480 + }, + { + "epoch": 0.88, + "learning_rate": 2.0694207549966345e-07, + "logits/chosen": -2.2417845726013184, + "logits/rejected": -2.1046087741851807, + "logps/chosen": -223.4938507080078, + "logps/rejected": -215.11630249023438, + "loss": 2201.9396, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14585386216640472, + "rewards/margins": 0.08491906523704529, + "rewards/rejected": -0.23077292740345, + "step": 13490 + }, + { + "epoch": 0.88, + "learning_rate": 2.0467337986423864e-07, + "logits/chosen": -2.474929094314575, + "logits/rejected": -2.1780519485473633, + "logps/chosen": -313.8457946777344, + "logps/rejected": -297.32183837890625, + "loss": 1621.7527, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08122257888317108, + "rewards/margins": 0.166888028383255, + "rewards/rejected": -0.2481106072664261, + "step": 13500 + }, + { + "epoch": 0.88, + "eval_logits/chosen": -2.3466689586639404, + "eval_logits/rejected": -2.159330368041992, + "eval_logps/chosen": -243.5742950439453, + "eval_logps/rejected": -238.83111572265625, + "eval_loss": 1753.5302734375, + "eval_rewards/accuracies": 0.6639999747276306, + "eval_rewards/chosen": -0.11569356173276901, + "eval_rewards/margins": 0.15649853646755219, + "eval_rewards/rejected": -0.2721921503543854, + "eval_runtime": 724.1227, + "eval_samples_per_second": 2.762, + "eval_steps_per_second": 1.381, + "step": 13500 + }, + { + "epoch": 0.88, + "learning_rate": 2.0241665777684272e-07, + "logits/chosen": -2.389523983001709, + "logits/rejected": -2.3051490783691406, + "logps/chosen": -275.768798828125, + "logps/rejected": -255.1609649658203, + "loss": 1251.2314, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07475773990154266, + "rewards/margins": 0.23007413744926453, + "rewards/rejected": -0.3048318922519684, + "step": 13510 + }, + { + "epoch": 0.88, + "learning_rate": 2.0017192100964366e-07, + "logits/chosen": -2.056959629058838, + "logits/rejected": -2.1443307399749756, + "logps/chosen": -214.5357208251953, + "logps/rejected": -231.7271270751953, + "loss": 1680.6148, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14579695463180542, + "rewards/margins": 0.15074291825294495, + "rewards/rejected": -0.296539843082428, + "step": 13520 + }, + { + "epoch": 0.89, + "learning_rate": 1.9793918127228777e-07, + "logits/chosen": -2.4231154918670654, + "logits/rejected": -2.0626559257507324, + "logps/chosen": -327.86822509765625, + "logps/rejected": -292.716552734375, + "loss": 1599.2062, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11568768322467804, + "rewards/margins": 0.18930375576019287, + "rewards/rejected": -0.3049914240837097, + "step": 13530 + }, + { + "epoch": 0.89, + "learning_rate": 1.9571845021184005e-07, + "logits/chosen": -2.2009639739990234, + "logits/rejected": -2.106663703918457, + "logps/chosen": -245.4452362060547, + "logps/rejected": -262.6272888183594, + "loss": 1927.0613, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14459358155727386, + "rewards/margins": 0.13064053654670715, + "rewards/rejected": -0.2752341032028198, + "step": 13540 + }, + { + "epoch": 0.89, + "learning_rate": 1.9350973941272027e-07, + "logits/chosen": -2.3240561485290527, + "logits/rejected": -2.3074772357940674, + "logps/chosen": -221.6490478515625, + "logps/rejected": -213.5911407470703, + "loss": 1959.1766, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1787400245666504, + "rewards/margins": 0.12442886829376221, + "rewards/rejected": -0.303168922662735, + "step": 13550 + }, + { + "epoch": 0.89, + "learning_rate": 1.9131306039664676e-07, + "logits/chosen": -2.201411008834839, + "logits/rejected": -2.1508049964904785, + "logps/chosen": -209.1273651123047, + "logps/rejected": -252.2547149658203, + "loss": 1552.3877, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09223302453756332, + "rewards/margins": 0.18770921230316162, + "rewards/rejected": -0.27994224429130554, + "step": 13560 + }, + { + "epoch": 0.89, + "learning_rate": 1.8912842462257358e-07, + "logits/chosen": -2.23241925239563, + "logits/rejected": -2.1687450408935547, + "logps/chosen": -229.80410766601562, + "logps/rejected": -234.39785766601562, + "loss": 1461.8519, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09634847939014435, + "rewards/margins": 0.21580667793750763, + "rewards/rejected": -0.312155157327652, + "step": 13570 + }, + { + "epoch": 0.89, + "learning_rate": 1.869558434866303e-07, + "logits/chosen": -2.3052706718444824, + "logits/rejected": -2.36867356300354, + "logps/chosen": -195.76123046875, + "logps/rejected": -230.22836303710938, + "loss": 1875.9637, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14671394228935242, + "rewards/margins": 0.16437289118766785, + "rewards/rejected": -0.31108683347702026, + "step": 13580 + }, + { + "epoch": 0.89, + "learning_rate": 1.847953283220652e-07, + "logits/chosen": -2.4721105098724365, + "logits/rejected": -2.1537983417510986, + "logps/chosen": -265.67724609375, + "logps/rejected": -210.75735473632812, + "loss": 1502.0496, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.0956752747297287, + "rewards/margins": 0.2371629923582077, + "rewards/rejected": -0.332838237285614, + "step": 13590 + }, + { + "epoch": 0.89, + "learning_rate": 1.8264689039918265e-07, + "logits/chosen": -2.440962553024292, + "logits/rejected": -2.101656436920166, + "logps/chosen": -273.24176025390625, + "logps/rejected": -256.9324951171875, + "loss": 2175.7262, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13411381840705872, + "rewards/margins": 0.11116938292980194, + "rewards/rejected": -0.24528317153453827, + "step": 13600 + }, + { + "epoch": 0.89, + "eval_logits/chosen": -2.345820665359497, + "eval_logits/rejected": -2.1585347652435303, + "eval_logps/chosen": -243.89022827148438, + "eval_logps/rejected": -239.07879638671875, + "eval_loss": 1754.93798828125, + "eval_rewards/accuracies": 0.6650000214576721, + "eval_rewards/chosen": -0.11885278671979904, + "eval_rewards/margins": 0.15581615269184113, + "eval_rewards/rejected": -0.2746689021587372, + "eval_runtime": 722.8485, + "eval_samples_per_second": 2.767, + "eval_steps_per_second": 1.383, + "step": 13600 + }, + { + "epoch": 0.89, + "learning_rate": 1.8051054092528857e-07, + "logits/chosen": -2.369603395462036, + "logits/rejected": -2.187791109085083, + "logps/chosen": -272.3529968261719, + "logps/rejected": -278.944580078125, + "loss": 1839.5477, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11251578480005264, + "rewards/margins": 0.1635177880525589, + "rewards/rejected": -0.27603358030319214, + "step": 13610 + }, + { + "epoch": 0.89, + "learning_rate": 1.783862910446271e-07, + "logits/chosen": -2.014069080352783, + "logits/rejected": -2.154714822769165, + "logps/chosen": -184.99124145507812, + "logps/rejected": -192.1890869140625, + "loss": 1712.8154, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12137937545776367, + "rewards/margins": 0.16512703895568848, + "rewards/rejected": -0.28650641441345215, + "step": 13620 + }, + { + "epoch": 0.89, + "learning_rate": 1.762741518383271e-07, + "logits/chosen": -2.3821609020233154, + "logits/rejected": -2.225235939025879, + "logps/chosen": -235.3060302734375, + "logps/rejected": -222.5266876220703, + "loss": 1856.7225, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13473662734031677, + "rewards/margins": 0.1488475352525711, + "rewards/rejected": -0.2835841178894043, + "step": 13630 + }, + { + "epoch": 0.89, + "learning_rate": 1.7417413432434082e-07, + "logits/chosen": -2.453998565673828, + "logits/rejected": -2.059866428375244, + "logps/chosen": -266.7665100097656, + "logps/rejected": -228.9359130859375, + "loss": 1901.8699, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1410680115222931, + "rewards/margins": 0.14936773478984833, + "rewards/rejected": -0.2904357612133026, + "step": 13640 + }, + { + "epoch": 0.89, + "learning_rate": 1.7208624945738855e-07, + "logits/chosen": -2.4492390155792236, + "logits/rejected": -2.297363758087158, + "logps/chosen": -230.3117218017578, + "logps/rejected": -250.00094604492188, + "loss": 1887.1781, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12227406352758408, + "rewards/margins": 0.12251828610897064, + "rewards/rejected": -0.24479234218597412, + "step": 13650 + }, + { + "epoch": 0.89, + "learning_rate": 1.7001050812889995e-07, + "logits/chosen": -2.42549467086792, + "logits/rejected": -2.1087913513183594, + "logps/chosen": -273.09515380859375, + "logps/rejected": -245.15701293945312, + "loss": 1939.2883, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17562313377857208, + "rewards/margins": 0.12312822043895721, + "rewards/rejected": -0.2987513840198517, + "step": 13660 + }, + { + "epoch": 0.89, + "learning_rate": 1.679469211669596e-07, + "logits/chosen": -2.3549065589904785, + "logits/rejected": -2.198781728744507, + "logps/chosen": -238.4629669189453, + "logps/rejected": -202.82814025878906, + "loss": 1538.0891, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.13517813384532928, + "rewards/margins": 0.159695565700531, + "rewards/rejected": -0.2948737144470215, + "step": 13670 + }, + { + "epoch": 0.9, + "learning_rate": 1.6589549933624715e-07, + "logits/chosen": -2.331890106201172, + "logits/rejected": -2.1530439853668213, + "logps/chosen": -252.07125854492188, + "logps/rejected": -222.9897918701172, + "loss": 1462.2755, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1186424270272255, + "rewards/margins": 0.22403955459594727, + "rewards/rejected": -0.34268200397491455, + "step": 13680 + }, + { + "epoch": 0.9, + "learning_rate": 1.638562533379845e-07, + "logits/chosen": -2.3359556198120117, + "logits/rejected": -2.1710317134857178, + "logps/chosen": -265.05255126953125, + "logps/rejected": -217.8121795654297, + "loss": 1876.0527, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0934990718960762, + "rewards/margins": 0.15027651190757751, + "rewards/rejected": -0.24377556145191193, + "step": 13690 + }, + { + "epoch": 0.9, + "learning_rate": 1.6182919380987676e-07, + "logits/chosen": -2.3831872940063477, + "logits/rejected": -2.301400661468506, + "logps/chosen": -238.47140502929688, + "logps/rejected": -232.7339630126953, + "loss": 2119.6848, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.12753424048423767, + "rewards/margins": 0.09639700502157211, + "rewards/rejected": -0.2239312380552292, + "step": 13700 + }, + { + "epoch": 0.9, + "eval_logits/chosen": -2.344073534011841, + "eval_logits/rejected": -2.15694260597229, + "eval_logps/chosen": -244.05441284179688, + "eval_logps/rejected": -239.03968811035156, + "eval_loss": 1754.270263671875, + "eval_rewards/accuracies": 0.6654999852180481, + "eval_rewards/chosen": -0.12049450725317001, + "eval_rewards/margins": 0.1537831723690033, + "eval_rewards/rejected": -0.2742776870727539, + "eval_runtime": 722.7989, + "eval_samples_per_second": 2.767, + "eval_steps_per_second": 1.384, + "step": 13700 + }, + { + "epoch": 0.9, + "learning_rate": 1.598143313260603e-07, + "logits/chosen": -2.3030202388763428, + "logits/rejected": -2.205902338027954, + "logps/chosen": -202.69361877441406, + "logps/rejected": -203.13424682617188, + "loss": 1727.1043, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1463984251022339, + "rewards/margins": 0.12455177307128906, + "rewards/rejected": -0.27095022797584534, + "step": 13710 + }, + { + "epoch": 0.9, + "learning_rate": 1.5781167639704415e-07, + "logits/chosen": -2.5292487144470215, + "logits/rejected": -1.9871896505355835, + "logps/chosen": -342.32806396484375, + "logps/rejected": -234.9232940673828, + "loss": 1738.8637, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10161248594522476, + "rewards/margins": 0.16838693618774414, + "rewards/rejected": -0.2699994444847107, + "step": 13720 + }, + { + "epoch": 0.9, + "learning_rate": 1.5582123946965787e-07, + "logits/chosen": -2.20857310295105, + "logits/rejected": -2.0954978466033936, + "logps/chosen": -247.48794555664062, + "logps/rejected": -272.25823974609375, + "loss": 1700.593, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1309293508529663, + "rewards/margins": 0.14622342586517334, + "rewards/rejected": -0.27715277671813965, + "step": 13730 + }, + { + "epoch": 0.9, + "learning_rate": 1.5384303092699504e-07, + "logits/chosen": -2.407092571258545, + "logits/rejected": -2.183859348297119, + "logps/chosen": -292.9111328125, + "logps/rejected": -317.5841064453125, + "loss": 1062.2133, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.05926471948623657, + "rewards/margins": 0.263541042804718, + "rewards/rejected": -0.322805792093277, + "step": 13740 + }, + { + "epoch": 0.9, + "learning_rate": 1.518770610883613e-07, + "logits/chosen": -2.2973599433898926, + "logits/rejected": -2.0243818759918213, + "logps/chosen": -228.43264770507812, + "logps/rejected": -219.4680633544922, + "loss": 1424.9246, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.13893964886665344, + "rewards/margins": 0.20449288189411163, + "rewards/rejected": -0.3434325158596039, + "step": 13750 + }, + { + "epoch": 0.9, + "learning_rate": 1.4992334020921735e-07, + "logits/chosen": -2.292459487915039, + "logits/rejected": -2.1910786628723145, + "logps/chosen": -180.3132781982422, + "logps/rejected": -172.5894775390625, + "loss": 1297.7433, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07340843975543976, + "rewards/margins": 0.21890044212341309, + "rewards/rejected": -0.29230886697769165, + "step": 13760 + }, + { + "epoch": 0.9, + "learning_rate": 1.4798187848112905e-07, + "logits/chosen": -2.2184767723083496, + "logits/rejected": -2.228837490081787, + "logps/chosen": -242.86160278320312, + "logps/rejected": -216.32040405273438, + "loss": 1840.3234, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.19890835881233215, + "rewards/margins": 0.14316312968730927, + "rewards/rejected": -0.3420714735984802, + "step": 13770 + }, + { + "epoch": 0.9, + "learning_rate": 1.460526860317113e-07, + "logits/chosen": -2.4182865619659424, + "logits/rejected": -2.3499302864074707, + "logps/chosen": -186.46209716796875, + "logps/rejected": -236.97122192382812, + "loss": 1507.47, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1317743957042694, + "rewards/margins": 0.1876230686903, + "rewards/rejected": -0.3193974494934082, + "step": 13780 + }, + { + "epoch": 0.9, + "learning_rate": 1.441357729245771e-07, + "logits/chosen": -2.5390188694000244, + "logits/rejected": -1.9643266201019287, + "logps/chosen": -268.406005859375, + "logps/rejected": -223.5176544189453, + "loss": 1951.6914, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2041414976119995, + "rewards/margins": 0.11098869889974594, + "rewards/rejected": -0.3151302635669708, + "step": 13790 + }, + { + "epoch": 0.9, + "learning_rate": 1.4223114915928482e-07, + "logits/chosen": -2.1964828968048096, + "logits/rejected": -1.9413315057754517, + "logps/chosen": -238.6462860107422, + "logps/rejected": -262.0941467285156, + "loss": 1448.3924, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.12723703682422638, + "rewards/margins": 0.1668069064617157, + "rewards/rejected": -0.29404395818710327, + "step": 13800 + }, + { + "epoch": 0.9, + "eval_logits/chosen": -2.345473289489746, + "eval_logits/rejected": -2.158222198486328, + "eval_logps/chosen": -243.75439453125, + "eval_logps/rejected": -238.92115783691406, + "eval_loss": 1754.062255859375, + "eval_rewards/accuracies": 0.6650000214576721, + "eval_rewards/chosen": -0.11749444156885147, + "eval_rewards/margins": 0.15559816360473633, + "eval_rewards/rejected": -0.2730926275253296, + "eval_runtime": 723.8834, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.381, + "step": 13800 + }, + { + "epoch": 0.9, + "learning_rate": 1.403388246712842e-07, + "logits/chosen": -2.2534501552581787, + "logits/rejected": -2.0139777660369873, + "logps/chosen": -177.90037536621094, + "logps/rejected": -181.38209533691406, + "loss": 1909.3896, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.13649320602416992, + "rewards/margins": 0.10633499920368195, + "rewards/rejected": -0.24282817542552948, + "step": 13810 + }, + { + "epoch": 0.9, + "learning_rate": 1.3845880933186757e-07, + "logits/chosen": -2.5113167762756348, + "logits/rejected": -2.246593952178955, + "logps/chosen": -249.75888061523438, + "logps/rejected": -226.8916778564453, + "loss": 1985.9889, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11020083725452423, + "rewards/margins": 0.11368707567453384, + "rewards/rejected": -0.22388792037963867, + "step": 13820 + }, + { + "epoch": 0.9, + "learning_rate": 1.3659111294811457e-07, + "logits/chosen": -2.345294237136841, + "logits/rejected": -2.2051594257354736, + "logps/chosen": -205.89266967773438, + "logps/rejected": -203.26119995117188, + "loss": 1693.6232, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1404949128627777, + "rewards/margins": 0.15264761447906494, + "rewards/rejected": -0.29314249753952026, + "step": 13830 + }, + { + "epoch": 0.91, + "learning_rate": 1.347357452628459e-07, + "logits/chosen": -2.4888830184936523, + "logits/rejected": -2.35544490814209, + "logps/chosen": -254.90744018554688, + "logps/rejected": -262.67059326171875, + "loss": 1875.2223, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09609898179769516, + "rewards/margins": 0.1526482105255127, + "rewards/rejected": -0.24874719977378845, + "step": 13840 + }, + { + "epoch": 0.91, + "learning_rate": 1.3289271595456732e-07, + "logits/chosen": -2.2717719078063965, + "logits/rejected": -2.0894250869750977, + "logps/chosen": -220.32534790039062, + "logps/rejected": -210.92330932617188, + "loss": 1640.549, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.15996167063713074, + "rewards/margins": 0.16391651332378387, + "rewards/rejected": -0.3238781988620758, + "step": 13850 + }, + { + "epoch": 0.91, + "learning_rate": 1.310620346374228e-07, + "logits/chosen": -2.1835789680480957, + "logits/rejected": -2.0586397647857666, + "logps/chosen": -243.33395385742188, + "logps/rejected": -223.63174438476562, + "loss": 1724.6162, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11112016439437866, + "rewards/margins": 0.1710091084241867, + "rewards/rejected": -0.2821292579174042, + "step": 13860 + }, + { + "epoch": 0.91, + "learning_rate": 1.2924371086114274e-07, + "logits/chosen": -2.2265589237213135, + "logits/rejected": -1.989508032798767, + "logps/chosen": -248.4755401611328, + "logps/rejected": -258.7160339355469, + "loss": 1779.283, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12414033710956573, + "rewards/margins": 0.1626734584569931, + "rewards/rejected": -0.28681379556655884, + "step": 13870 + }, + { + "epoch": 0.91, + "learning_rate": 1.274377541109953e-07, + "logits/chosen": -2.1988863945007324, + "logits/rejected": -2.268820285797119, + "logps/chosen": -178.72723388671875, + "logps/rejected": -268.6763610839844, + "loss": 1886.5996, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.14376714825630188, + "rewards/margins": 0.12025441229343414, + "rewards/rejected": -0.2640215754508972, + "step": 13880 + }, + { + "epoch": 0.91, + "learning_rate": 1.2564417380773435e-07, + "logits/chosen": -2.1269640922546387, + "logits/rejected": -1.9930492639541626, + "logps/chosen": -192.80255126953125, + "logps/rejected": -233.09622192382812, + "loss": 1588.2561, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14305493235588074, + "rewards/margins": 0.178117036819458, + "rewards/rejected": -0.32117193937301636, + "step": 13890 + }, + { + "epoch": 0.91, + "learning_rate": 1.2386297930755436e-07, + "logits/chosen": -2.337871789932251, + "logits/rejected": -2.3502695560455322, + "logps/chosen": -267.2091064453125, + "logps/rejected": -277.0139465332031, + "loss": 1953.3191, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1932331919670105, + "rewards/margins": 0.1463126540184021, + "rewards/rejected": -0.3395458161830902, + "step": 13900 + }, + { + "epoch": 0.91, + "eval_logits/chosen": -2.3461110591888428, + "eval_logits/rejected": -2.1588072776794434, + "eval_logps/chosen": -243.5794677734375, + "eval_logps/rejected": -238.8184051513672, + "eval_loss": 1754.0230712890625, + "eval_rewards/accuracies": 0.6650000214576721, + "eval_rewards/chosen": -0.1157451868057251, + "eval_rewards/margins": 0.15631982684135437, + "eval_rewards/rejected": -0.27206501364707947, + "eval_runtime": 725.5684, + "eval_samples_per_second": 2.756, + "eval_steps_per_second": 1.378, + "step": 13900 + }, + { + "epoch": 0.91, + "learning_rate": 1.220941799020378e-07, + "logits/chosen": -2.123575448989868, + "logits/rejected": -2.0426182746887207, + "logps/chosen": -230.9011688232422, + "logps/rejected": -225.5489959716797, + "loss": 1631.8912, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11820058524608612, + "rewards/margins": 0.17000697553157806, + "rewards/rejected": -0.2882075905799866, + "step": 13910 + }, + { + "epoch": 0.91, + "learning_rate": 1.2033778481810975e-07, + "logits/chosen": -2.401193380355835, + "logits/rejected": -2.134850263595581, + "logps/chosen": -230.31613159179688, + "logps/rejected": -210.4309844970703, + "loss": 1748.4959, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10322584211826324, + "rewards/margins": 0.17461058497428894, + "rewards/rejected": -0.277836412191391, + "step": 13920 + }, + { + "epoch": 0.91, + "learning_rate": 1.1859380321798591e-07, + "logits/chosen": -2.343634605407715, + "logits/rejected": -2.407421588897705, + "logps/chosen": -216.51547241210938, + "logps/rejected": -242.3054656982422, + "loss": 1944.4545, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1395423412322998, + "rewards/margins": 0.11045052856206894, + "rewards/rejected": -0.24999287724494934, + "step": 13930 + }, + { + "epoch": 0.91, + "learning_rate": 1.1686224419912989e-07, + "logits/chosen": -2.2427189350128174, + "logits/rejected": -2.036081314086914, + "logps/chosen": -264.5870666503906, + "logps/rejected": -256.62005615234375, + "loss": 1467.0031, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1171443909406662, + "rewards/margins": 0.21323764324188232, + "rewards/rejected": -0.33038201928138733, + "step": 13940 + }, + { + "epoch": 0.91, + "learning_rate": 1.1514311679420104e-07, + "logits/chosen": -2.0678160190582275, + "logits/rejected": -2.1402063369750977, + "logps/chosen": -172.3576202392578, + "logps/rejected": -238.7569122314453, + "loss": 2250.3074, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17209061980247498, + "rewards/margins": 0.10579583793878555, + "rewards/rejected": -0.27788645029067993, + "step": 13950 + }, + { + "epoch": 0.91, + "learning_rate": 1.1343642997101029e-07, + "logits/chosen": -2.3408255577087402, + "logits/rejected": -2.248295307159424, + "logps/chosen": -213.69570922851562, + "logps/rejected": -214.92318725585938, + "loss": 1828.8832, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11342382431030273, + "rewards/margins": 0.1444929540157318, + "rewards/rejected": -0.25791677832603455, + "step": 13960 + }, + { + "epoch": 0.91, + "learning_rate": 1.1174219263247188e-07, + "logits/chosen": -2.0945498943328857, + "logits/rejected": -1.986140251159668, + "logps/chosen": -211.49118041992188, + "logps/rejected": -206.6245880126953, + "loss": 1908.091, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12673640251159668, + "rewards/margins": 0.12664535641670227, + "rewards/rejected": -0.25338175892829895, + "step": 13970 + }, + { + "epoch": 0.91, + "learning_rate": 1.1006041361655839e-07, + "logits/chosen": -2.5157954692840576, + "logits/rejected": -2.055115222930908, + "logps/chosen": -220.3848419189453, + "logps/rejected": -195.51856994628906, + "loss": 1765.3135, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11947546899318695, + "rewards/margins": 0.15071254968643188, + "rewards/rejected": -0.27018803358078003, + "step": 13980 + }, + { + "epoch": 0.92, + "learning_rate": 1.0839110169625189e-07, + "logits/chosen": -2.0732357501983643, + "logits/rejected": -2.3762660026550293, + "logps/chosen": -222.16049194335938, + "logps/rejected": -235.0498046875, + "loss": 1506.5421, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15318220853805542, + "rewards/margins": 0.2024463713169098, + "rewards/rejected": -0.3556285500526428, + "step": 13990 + }, + { + "epoch": 0.92, + "learning_rate": 1.06734265579502e-07, + "logits/chosen": -2.355957508087158, + "logits/rejected": -2.0589470863342285, + "logps/chosen": -274.600830078125, + "logps/rejected": -224.1596221923828, + "loss": 1684.251, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13084113597869873, + "rewards/margins": 0.1606401652097702, + "rewards/rejected": -0.2914813160896301, + "step": 14000 + }, + { + "epoch": 0.92, + "eval_logits/chosen": -2.3463921546936035, + "eval_logits/rejected": -2.1591100692749023, + "eval_logps/chosen": -243.4635772705078, + "eval_logps/rejected": -238.63746643066406, + "eval_loss": 1754.74755859375, + "eval_rewards/accuracies": 0.6635000109672546, + "eval_rewards/chosen": -0.11458618193864822, + "eval_rewards/margins": 0.15566939115524292, + "eval_rewards/rejected": -0.27025559544563293, + "eval_runtime": 722.042, + "eval_samples_per_second": 2.77, + "eval_steps_per_second": 1.385, + "step": 14000 + }, + { + "epoch": 0.92, + "learning_rate": 1.050899139091771e-07, + "logits/chosen": -2.4245903491973877, + "logits/rejected": -2.08056378364563, + "logps/chosen": -289.9082336425781, + "logps/rejected": -264.00091552734375, + "loss": 1234.4372, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11280794441699982, + "rewards/margins": 0.2061082124710083, + "rewards/rejected": -0.3189161419868469, + "step": 14010 + }, + { + "epoch": 0.92, + "learning_rate": 1.0345805526302072e-07, + "logits/chosen": -2.261943817138672, + "logits/rejected": -2.374112129211426, + "logps/chosen": -212.29641723632812, + "logps/rejected": -223.0238037109375, + "loss": 1472.3037, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.11134089529514313, + "rewards/margins": 0.17965057492256165, + "rewards/rejected": -0.29099148511886597, + "step": 14020 + }, + { + "epoch": 0.92, + "learning_rate": 1.0183869815360764e-07, + "logits/chosen": -2.2483181953430176, + "logits/rejected": -2.3696656227111816, + "logps/chosen": -203.56878662109375, + "logps/rejected": -250.1732940673828, + "loss": 1821.0598, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11655312776565552, + "rewards/margins": 0.13964414596557617, + "rewards/rejected": -0.2561972737312317, + "step": 14030 + }, + { + "epoch": 0.92, + "learning_rate": 1.0023185102829763e-07, + "logits/chosen": -2.061488389968872, + "logits/rejected": -2.2805399894714355, + "logps/chosen": -243.485595703125, + "logps/rejected": -264.13775634765625, + "loss": 1707.124, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.126633882522583, + "rewards/margins": 0.16467411816120148, + "rewards/rejected": -0.2913079857826233, + "step": 14040 + }, + { + "epoch": 0.92, + "learning_rate": 9.863752226919182e-08, + "logits/chosen": -2.27036452293396, + "logits/rejected": -1.7793105840682983, + "logps/chosen": -247.064453125, + "logps/rejected": -203.147705078125, + "loss": 1475.9483, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09332261979579926, + "rewards/margins": 0.2166186273097992, + "rewards/rejected": -0.30994123220443726, + "step": 14050 + }, + { + "epoch": 0.92, + "learning_rate": 9.705572019309107e-08, + "logits/chosen": -2.187352180480957, + "logits/rejected": -2.2290992736816406, + "logps/chosen": -281.07415771484375, + "logps/rejected": -265.98870849609375, + "loss": 1752.7209, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.13046763837337494, + "rewards/margins": 0.1639849692583084, + "rewards/rejected": -0.29445260763168335, + "step": 14060 + }, + { + "epoch": 0.92, + "learning_rate": 9.548645305144849e-08, + "logits/chosen": -2.4042441844940186, + "logits/rejected": -2.273912191390991, + "logps/chosen": -183.91830444335938, + "logps/rejected": -204.17153930664062, + "loss": 1756.1881, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.10423145443201065, + "rewards/margins": 0.16279512643814087, + "rewards/rejected": -0.2670265734195709, + "step": 14070 + }, + { + "epoch": 0.92, + "learning_rate": 9.392972903033149e-08, + "logits/chosen": -2.330324649810791, + "logits/rejected": -2.176776170730591, + "logps/chosen": -232.614013671875, + "logps/rejected": -234.63394165039062, + "loss": 2016.0381, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.09482771158218384, + "rewards/margins": 0.09563405811786652, + "rewards/rejected": -0.19046176970005035, + "step": 14080 + }, + { + "epoch": 0.92, + "learning_rate": 9.238555625037449e-08, + "logits/chosen": -2.350982189178467, + "logits/rejected": -2.135450839996338, + "logps/chosen": -203.74049377441406, + "logps/rejected": -186.6983184814453, + "loss": 1871.374, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1303228735923767, + "rewards/margins": 0.12639638781547546, + "rewards/rejected": -0.25671929121017456, + "step": 14090 + }, + { + "epoch": 0.92, + "learning_rate": 9.085394276673903e-08, + "logits/chosen": -2.3481802940368652, + "logits/rejected": -2.045968770980835, + "logps/chosen": -280.8945007324219, + "logps/rejected": -282.7396240234375, + "loss": 1545.3156, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.12728367745876312, + "rewards/margins": 0.16294625401496887, + "rewards/rejected": -0.2902299463748932, + "step": 14100 + }, + { + "epoch": 0.92, + "eval_logits/chosen": -2.3445770740509033, + "eval_logits/rejected": -2.157391309738159, + "eval_logps/chosen": -243.55126953125, + "eval_logps/rejected": -238.72972106933594, + "eval_loss": 1755.1064453125, + "eval_rewards/accuracies": 0.6650000214576721, + "eval_rewards/chosen": -0.11546298116445541, + "eval_rewards/margins": 0.15571531653404236, + "eval_rewards/rejected": -0.27117830514907837, + "eval_runtime": 725.3595, + "eval_samples_per_second": 2.757, + "eval_steps_per_second": 1.379, + "step": 14100 + }, + { + "epoch": 0.92, + "learning_rate": 8.933489656907157e-08, + "logits/chosen": -2.3288893699645996, + "logits/rejected": -2.2169697284698486, + "logps/chosen": -231.6782684326172, + "logps/rejected": -265.7174072265625, + "loss": 2114.174, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.14342829585075378, + "rewards/margins": 0.09098762273788452, + "rewards/rejected": -0.2344159185886383, + "step": 14110 + }, + { + "epoch": 0.92, + "learning_rate": 8.782842558146127e-08, + "logits/chosen": -2.391118288040161, + "logits/rejected": -2.3201541900634766, + "logps/chosen": -170.07455444335938, + "logps/rejected": -187.50924682617188, + "loss": 1380.3342, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.08190619945526123, + "rewards/margins": 0.1914120465517044, + "rewards/rejected": -0.2733182907104492, + "step": 14120 + }, + { + "epoch": 0.92, + "learning_rate": 8.633453766239836e-08, + "logits/chosen": -2.4329991340637207, + "logits/rejected": -2.2017290592193604, + "logps/chosen": -243.4434356689453, + "logps/rejected": -222.77706909179688, + "loss": 1938.3652, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08322300016880035, + "rewards/margins": 0.11188875138759613, + "rewards/rejected": -0.19511176645755768, + "step": 14130 + }, + { + "epoch": 0.93, + "learning_rate": 8.485324060473448e-08, + "logits/chosen": -2.2668330669403076, + "logits/rejected": -2.1573243141174316, + "logps/chosen": -252.8908233642578, + "logps/rejected": -254.57229614257812, + "loss": 1929.8533, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12608733773231506, + "rewards/margins": 0.14072270691394806, + "rewards/rejected": -0.26681002974510193, + "step": 14140 + }, + { + "epoch": 0.93, + "learning_rate": 8.338454213564052e-08, + "logits/chosen": -2.3068039417266846, + "logits/rejected": -2.0356509685516357, + "logps/chosen": -242.44216918945312, + "logps/rejected": -246.11767578125, + "loss": 1416.0274, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.11545213311910629, + "rewards/margins": 0.2086973637342453, + "rewards/rejected": -0.3241495192050934, + "step": 14150 + }, + { + "epoch": 0.93, + "learning_rate": 8.192844991656679e-08, + "logits/chosen": -2.3066954612731934, + "logits/rejected": -2.0747666358947754, + "logps/chosen": -250.7014617919922, + "logps/rejected": -228.00930786132812, + "loss": 1824.4467, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11984783411026001, + "rewards/margins": 0.13715848326683044, + "rewards/rejected": -0.25700634717941284, + "step": 14160 + }, + { + "epoch": 0.93, + "learning_rate": 8.048497154320434e-08, + "logits/chosen": -2.3507096767425537, + "logits/rejected": -2.395566463470459, + "logps/chosen": -146.2378692626953, + "logps/rejected": -162.6152801513672, + "loss": 2089.841, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1668834388256073, + "rewards/margins": 0.07279939204454422, + "rewards/rejected": -0.23968283832073212, + "step": 14170 + }, + { + "epoch": 0.93, + "learning_rate": 7.905411454544265e-08, + "logits/chosen": -2.3429198265075684, + "logits/rejected": -2.190000295639038, + "logps/chosen": -250.3459014892578, + "logps/rejected": -266.3189392089844, + "loss": 2110.5916, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1384165734052658, + "rewards/margins": 0.11230772733688354, + "rewards/rejected": -0.25072425603866577, + "step": 14180 + }, + { + "epoch": 0.93, + "learning_rate": 7.763588638733332e-08, + "logits/chosen": -2.357968807220459, + "logits/rejected": -2.3119089603424072, + "logps/chosen": -275.1587829589844, + "logps/rejected": -268.12847900390625, + "loss": 1896.2953, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12738670408725739, + "rewards/margins": 0.12793752551078796, + "rewards/rejected": -0.25532421469688416, + "step": 14190 + }, + { + "epoch": 0.93, + "learning_rate": 7.623029446704899e-08, + "logits/chosen": -2.21120023727417, + "logits/rejected": -2.3920819759368896, + "logps/chosen": -319.13970947265625, + "logps/rejected": -307.9252014160156, + "loss": 1391.3224, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.07698237895965576, + "rewards/margins": 0.2568013072013855, + "rewards/rejected": -0.33378368616104126, + "step": 14200 + }, + { + "epoch": 0.93, + "eval_logits/chosen": -2.343921184539795, + "eval_logits/rejected": -2.1568198204040527, + "eval_logps/chosen": -243.383056640625, + "eval_logps/rejected": -238.62802124023438, + "eval_loss": 1756.5655517578125, + "eval_rewards/accuracies": 0.6614999771118164, + "eval_rewards/chosen": -0.11378104984760284, + "eval_rewards/margins": 0.15638026595115662, + "eval_rewards/rejected": -0.2701612710952759, + "eval_runtime": 724.3082, + "eval_samples_per_second": 2.761, + "eval_steps_per_second": 1.381, + "step": 14200 + }, + { + "epoch": 0.93, + "learning_rate": 7.483734611684557e-08, + "logits/chosen": -2.137566089630127, + "logits/rejected": -1.9804027080535889, + "logps/chosen": -263.27410888671875, + "logps/rejected": -228.4842987060547, + "loss": 1619.2999, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.063882976770401, + "rewards/margins": 0.19371116161346436, + "rewards/rejected": -0.25759413838386536, + "step": 14210 + }, + { + "epoch": 0.93, + "learning_rate": 7.345704860302366e-08, + "logits/chosen": -2.4090185165405273, + "logits/rejected": -2.4050068855285645, + "logps/chosen": -257.5478515625, + "logps/rejected": -277.10076904296875, + "loss": 1403.6926, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09295176714658737, + "rewards/margins": 0.20231318473815918, + "rewards/rejected": -0.29526492953300476, + "step": 14220 + }, + { + "epoch": 0.93, + "learning_rate": 7.208940912589224e-08, + "logits/chosen": -2.3491759300231934, + "logits/rejected": -2.019956350326538, + "logps/chosen": -223.9944610595703, + "logps/rejected": -205.76278686523438, + "loss": 1605.3204, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1440814882516861, + "rewards/margins": 0.19146788120269775, + "rewards/rejected": -0.33554935455322266, + "step": 14230 + }, + { + "epoch": 0.93, + "learning_rate": 7.073443481972753e-08, + "logits/chosen": -2.1687004566192627, + "logits/rejected": -2.1498425006866455, + "logps/chosen": -200.83572387695312, + "logps/rejected": -232.8833770751953, + "loss": 2132.1936, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.16679982841014862, + "rewards/margins": 0.12545950710773468, + "rewards/rejected": -0.2922593355178833, + "step": 14240 + }, + { + "epoch": 0.93, + "learning_rate": 6.939213275274027e-08, + "logits/chosen": -2.2963318824768066, + "logits/rejected": -2.25905179977417, + "logps/chosen": -251.19442749023438, + "logps/rejected": -244.81991577148438, + "loss": 1795.9357, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11940628290176392, + "rewards/margins": 0.13228197395801544, + "rewards/rejected": -0.25168827176094055, + "step": 14250 + }, + { + "epoch": 0.93, + "learning_rate": 6.806250992703461e-08, + "logits/chosen": -2.3213610649108887, + "logits/rejected": -2.1692914962768555, + "logps/chosen": -225.44091796875, + "logps/rejected": -215.24996948242188, + "loss": 1622.4871, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08746316283941269, + "rewards/margins": 0.16019216179847717, + "rewards/rejected": -0.24765534698963165, + "step": 14260 + }, + { + "epoch": 0.93, + "learning_rate": 6.674557327857572e-08, + "logits/chosen": -2.313218593597412, + "logits/rejected": -2.326613664627075, + "logps/chosen": -266.9938659667969, + "logps/rejected": -275.0752258300781, + "loss": 1828.6219, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12723203003406525, + "rewards/margins": 0.17169377207756042, + "rewards/rejected": -0.29892581701278687, + "step": 14270 + }, + { + "epoch": 0.93, + "learning_rate": 6.544132967714917e-08, + "logits/chosen": -2.0654211044311523, + "logits/rejected": -2.0778846740722656, + "logps/chosen": -262.28765869140625, + "logps/rejected": -264.19000244140625, + "loss": 2270.5215, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15427663922309875, + "rewards/margins": 0.15614831447601318, + "rewards/rejected": -0.31042495369911194, + "step": 14280 + }, + { + "epoch": 0.93, + "learning_rate": 6.414978592632932e-08, + "logits/chosen": -2.394740104675293, + "logits/rejected": -1.9486885070800781, + "logps/chosen": -274.8138732910156, + "logps/rejected": -241.82949829101562, + "loss": 1756.9453, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1371452510356903, + "rewards/margins": 0.13471072912216187, + "rewards/rejected": -0.2718559801578522, + "step": 14290 + }, + { + "epoch": 0.94, + "learning_rate": 6.287094876344046e-08, + "logits/chosen": -2.3430347442626953, + "logits/rejected": -2.3814728260040283, + "logps/chosen": -179.88137817382812, + "logps/rejected": -201.48204040527344, + "loss": 1588.1222, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.07461106032133102, + "rewards/margins": 0.14961650967597961, + "rewards/rejected": -0.22422757744789124, + "step": 14300 + }, + { + "epoch": 0.94, + "eval_logits/chosen": -2.345165967941284, + "eval_logits/rejected": -2.157975196838379, + "eval_logps/chosen": -243.50830078125, + "eval_logps/rejected": -238.7296142578125, + "eval_loss": 1755.2308349609375, + "eval_rewards/accuracies": 0.6650000214576721, + "eval_rewards/chosen": -0.1150333434343338, + "eval_rewards/margins": 0.15614359080791473, + "eval_rewards/rejected": -0.27117693424224854, + "eval_runtime": 723.0581, + "eval_samples_per_second": 2.766, + "eval_steps_per_second": 1.383, + "step": 14300 + }, + { + "epoch": 0.94, + "learning_rate": 6.160482485952413e-08, + "logits/chosen": -2.4855408668518066, + "logits/rejected": -2.2154054641723633, + "logps/chosen": -248.1533660888672, + "logps/rejected": -232.2559814453125, + "loss": 1629.3428, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11703590303659439, + "rewards/margins": 0.15898486971855164, + "rewards/rejected": -0.27602076530456543, + "step": 14310 + }, + { + "epoch": 0.94, + "learning_rate": 6.035142081930234e-08, + "logits/chosen": -2.3523106575012207, + "logits/rejected": -1.9807660579681396, + "logps/chosen": -278.71728515625, + "logps/rejected": -214.4222412109375, + "loss": 1711.0033, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15920953452587128, + "rewards/margins": 0.12721839547157288, + "rewards/rejected": -0.28642791509628296, + "step": 14320 + }, + { + "epoch": 0.94, + "learning_rate": 5.911074318114496e-08, + "logits/chosen": -2.1858577728271484, + "logits/rejected": -2.3020291328430176, + "logps/chosen": -218.28091430664062, + "logps/rejected": -275.77850341796875, + "loss": 1649.2277, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14692769944667816, + "rewards/margins": 0.15372733771800995, + "rewards/rejected": -0.3006550669670105, + "step": 14330 + }, + { + "epoch": 0.94, + "learning_rate": 5.788279841703381e-08, + "logits/chosen": -2.386441707611084, + "logits/rejected": -2.135468006134033, + "logps/chosen": -195.0791015625, + "logps/rejected": -205.4237823486328, + "loss": 1574.9247, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11169525235891342, + "rewards/margins": 0.1801174432039261, + "rewards/rejected": -0.2918127179145813, + "step": 14340 + }, + { + "epoch": 0.94, + "learning_rate": 5.66675929325311e-08, + "logits/chosen": -2.3844265937805176, + "logits/rejected": -2.1434741020202637, + "logps/chosen": -230.9381103515625, + "logps/rejected": -237.17324829101562, + "loss": 1654.5648, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10273660719394684, + "rewards/margins": 0.17583905160427094, + "rewards/rejected": -0.278575599193573, + "step": 14350 + }, + { + "epoch": 0.94, + "learning_rate": 5.546513306674301e-08, + "logits/chosen": -2.3126797676086426, + "logits/rejected": -1.9176400899887085, + "logps/chosen": -291.2501525878906, + "logps/rejected": -231.6427459716797, + "loss": 1717.066, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.127334862947464, + "rewards/margins": 0.15475480258464813, + "rewards/rejected": -0.2820896506309509, + "step": 14360 + }, + { + "epoch": 0.94, + "learning_rate": 5.4275425092290004e-08, + "logits/chosen": -2.4252126216888428, + "logits/rejected": -2.3836467266082764, + "logps/chosen": -263.10675048828125, + "logps/rejected": -262.96209716796875, + "loss": 1506.7836, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.08310611546039581, + "rewards/margins": 0.1829802691936493, + "rewards/rejected": -0.2660863399505615, + "step": 14370 + }, + { + "epoch": 0.94, + "learning_rate": 5.309847521527078e-08, + "logits/chosen": -2.289580821990967, + "logits/rejected": -1.9175958633422852, + "logps/chosen": -294.80340576171875, + "logps/rejected": -267.6952819824219, + "loss": 1548.7974, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10230030119419098, + "rewards/margins": 0.15820232033729553, + "rewards/rejected": -0.2605026364326477, + "step": 14380 + }, + { + "epoch": 0.94, + "learning_rate": 5.1934289575233385e-08, + "logits/chosen": -2.164163112640381, + "logits/rejected": -1.8297512531280518, + "logps/chosen": -254.22085571289062, + "logps/rejected": -234.9806671142578, + "loss": 1854.2094, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.14691409468650818, + "rewards/margins": 0.15127232670783997, + "rewards/rejected": -0.29818639159202576, + "step": 14390 + }, + { + "epoch": 0.94, + "learning_rate": 5.078287424513994e-08, + "logits/chosen": -2.4062767028808594, + "logits/rejected": -2.2998061180114746, + "logps/chosen": -285.44677734375, + "logps/rejected": -225.17855834960938, + "loss": 1734.7881, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15399052202701569, + "rewards/margins": 0.14562062919139862, + "rewards/rejected": -0.2996111512184143, + "step": 14400 + }, + { + "epoch": 0.94, + "eval_logits/chosen": -2.3442108631134033, + "eval_logits/rejected": -2.1570913791656494, + "eval_logps/chosen": -243.45433044433594, + "eval_logps/rejected": -238.75839233398438, + "eval_loss": 1755.41015625, + "eval_rewards/accuracies": 0.6660000085830688, + "eval_rewards/chosen": -0.11449379473924637, + "eval_rewards/margins": 0.15697112679481506, + "eval_rewards/rejected": -0.27146491408348083, + "eval_runtime": 726.2683, + "eval_samples_per_second": 2.754, + "eval_steps_per_second": 1.377, + "step": 14400 + }, + { + "epoch": 0.94, + "learning_rate": 4.964423523133671e-08, + "logits/chosen": -2.4076712131500244, + "logits/rejected": -2.1759860515594482, + "logps/chosen": -225.8399200439453, + "logps/rejected": -205.88589477539062, + "loss": 1570.1146, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.08108416944742203, + "rewards/margins": 0.14395181834697723, + "rewards/rejected": -0.22503598034381866, + "step": 14410 + }, + { + "epoch": 0.94, + "learning_rate": 4.8518378473522976e-08, + "logits/chosen": -2.2773380279541016, + "logits/rejected": -2.1288530826568604, + "logps/chosen": -261.8817138671875, + "logps/rejected": -267.4660949707031, + "loss": 2089.3795, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.1137181892991066, + "rewards/margins": 0.12311627715826035, + "rewards/rejected": -0.23683448135852814, + "step": 14420 + }, + { + "epoch": 0.94, + "learning_rate": 4.7405309844718584e-08, + "logits/chosen": -2.20715594291687, + "logits/rejected": -2.104396343231201, + "logps/chosen": -209.18472290039062, + "logps/rejected": -236.4004364013672, + "loss": 1657.6336, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16685688495635986, + "rewards/margins": 0.18019473552703857, + "rewards/rejected": -0.34705162048339844, + "step": 14430 + }, + { + "epoch": 0.94, + "learning_rate": 4.630503515123508e-08, + "logits/chosen": -2.451308250427246, + "logits/rejected": -2.1538302898406982, + "logps/chosen": -213.4972686767578, + "logps/rejected": -178.93968200683594, + "loss": 1720.5678, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1263124644756317, + "rewards/margins": 0.15308813750743866, + "rewards/rejected": -0.27940061688423157, + "step": 14440 + }, + { + "epoch": 0.95, + "learning_rate": 4.5217560132644056e-08, + "logits/chosen": -2.268876314163208, + "logits/rejected": -2.199364185333252, + "logps/chosen": -160.91078186035156, + "logps/rejected": -193.87796020507812, + "loss": 1908.1941, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10291179269552231, + "rewards/margins": 0.14566278457641602, + "rewards/rejected": -0.24857458472251892, + "step": 14450 + }, + { + "epoch": 0.95, + "learning_rate": 4.41428904617483e-08, + "logits/chosen": -2.3149216175079346, + "logits/rejected": -2.306036949157715, + "logps/chosen": -189.7136993408203, + "logps/rejected": -209.6298828125, + "loss": 1795.1039, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12605008482933044, + "rewards/margins": 0.15255668759346008, + "rewards/rejected": -0.2786068022251129, + "step": 14460 + }, + { + "epoch": 0.95, + "learning_rate": 4.3081031744550696e-08, + "logits/chosen": -2.39857816696167, + "logits/rejected": -2.325693130493164, + "logps/chosen": -260.35296630859375, + "logps/rejected": -256.3681945800781, + "loss": 1418.1006, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07207634299993515, + "rewards/margins": 0.18342944979667664, + "rewards/rejected": -0.25550583004951477, + "step": 14470 + }, + { + "epoch": 0.95, + "learning_rate": 4.2031989520227025e-08, + "logits/chosen": -2.3827738761901855, + "logits/rejected": -2.1851248741149902, + "logps/chosen": -236.3775177001953, + "logps/rejected": -232.0126953125, + "loss": 1543.5579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08063056319952011, + "rewards/margins": 0.17522695660591125, + "rewards/rejected": -0.25585752725601196, + "step": 14480 + }, + { + "epoch": 0.95, + "learning_rate": 4.099576926109461e-08, + "logits/chosen": -2.4595134258270264, + "logits/rejected": -1.9548670053482056, + "logps/chosen": -250.75570678710938, + "logps/rejected": -188.27928161621094, + "loss": 1706.1455, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10973851382732391, + "rewards/margins": 0.16678059101104736, + "rewards/rejected": -0.27651911973953247, + "step": 14490 + }, + { + "epoch": 0.95, + "learning_rate": 3.997237637258705e-08, + "logits/chosen": -2.2871203422546387, + "logits/rejected": -2.351755142211914, + "logps/chosen": -324.89862060546875, + "logps/rejected": -300.61224365234375, + "loss": 1655.0535, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07645292580127716, + "rewards/margins": 0.19056789577007294, + "rewards/rejected": -0.2670208513736725, + "step": 14500 + }, + { + "epoch": 0.95, + "eval_logits/chosen": -2.3447322845458984, + "eval_logits/rejected": -2.157543659210205, + "eval_logps/chosen": -243.40638732910156, + "eval_logps/rejected": -238.72279357910156, + "eval_loss": 1754.942626953125, + "eval_rewards/accuracies": 0.6629999876022339, + "eval_rewards/chosen": -0.11401434987783432, + "eval_rewards/margins": 0.1570945829153061, + "eval_rewards/rejected": -0.2711089551448822, + "eval_runtime": 723.1645, + "eval_samples_per_second": 2.766, + "eval_steps_per_second": 1.383, + "step": 14500 + }, + { + "epoch": 0.95, + "learning_rate": 3.8961816193222035e-08, + "logits/chosen": -2.434767246246338, + "logits/rejected": -2.2198853492736816, + "logps/chosen": -252.49307250976562, + "logps/rejected": -203.490234375, + "loss": 2298.1338, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16642776131629944, + "rewards/margins": 0.0833040177822113, + "rewards/rejected": -0.24973177909851074, + "step": 14510 + }, + { + "epoch": 0.95, + "learning_rate": 3.79640939945769e-08, + "logits/chosen": -2.3867263793945312, + "logits/rejected": -2.262131452560425, + "logps/chosen": -291.5511169433594, + "logps/rejected": -218.9336700439453, + "loss": 1704.7814, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.05330218747258186, + "rewards/margins": 0.13646642863750458, + "rewards/rejected": -0.18976862728595734, + "step": 14520 + }, + { + "epoch": 0.95, + "learning_rate": 3.697921498125895e-08, + "logits/chosen": -2.1315157413482666, + "logits/rejected": -2.2267227172851562, + "logps/chosen": -223.8726043701172, + "logps/rejected": -243.6025390625, + "loss": 2002.6312, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.13919594883918762, + "rewards/margins": 0.13850137591362, + "rewards/rejected": -0.27769729495048523, + "step": 14530 + }, + { + "epoch": 0.95, + "learning_rate": 3.6007184290880456e-08, + "logits/chosen": -2.3476033210754395, + "logits/rejected": -2.250824451446533, + "logps/chosen": -227.66970825195312, + "logps/rejected": -225.08633422851562, + "loss": 2114.7318, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.15696708858013153, + "rewards/margins": 0.14080265164375305, + "rewards/rejected": -0.2977697253227234, + "step": 14540 + }, + { + "epoch": 0.95, + "learning_rate": 3.504800699402872e-08, + "logits/chosen": -2.5682365894317627, + "logits/rejected": -2.2721850872039795, + "logps/chosen": -356.9365539550781, + "logps/rejected": -299.3744812011719, + "loss": 1899.5432, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1128232479095459, + "rewards/margins": 0.15804831683635712, + "rewards/rejected": -0.2708715796470642, + "step": 14550 + }, + { + "epoch": 0.95, + "learning_rate": 3.4101688094242967e-08, + "logits/chosen": -2.28175687789917, + "logits/rejected": -2.1759145259857178, + "logps/chosen": -283.11199951171875, + "logps/rejected": -277.6907958984375, + "loss": 1573.1848, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1355103850364685, + "rewards/margins": 0.19754980504512787, + "rewards/rejected": -0.33306020498275757, + "step": 14560 + }, + { + "epoch": 0.95, + "learning_rate": 3.3168232527985564e-08, + "logits/chosen": -2.2301955223083496, + "logits/rejected": -1.9302619695663452, + "logps/chosen": -250.8253173828125, + "logps/rejected": -215.4320068359375, + "loss": 1733.818, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09206750243902206, + "rewards/margins": 0.19356587529182434, + "rewards/rejected": -0.285633385181427, + "step": 14570 + }, + { + "epoch": 0.95, + "learning_rate": 3.224764516461892e-08, + "logits/chosen": -2.3313794136047363, + "logits/rejected": -2.1601405143737793, + "logps/chosen": -265.72125244140625, + "logps/rejected": -253.18911743164062, + "loss": 1621.6596, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10722391307353973, + "rewards/margins": 0.18216519057750702, + "rewards/rejected": -0.28938913345336914, + "step": 14580 + }, + { + "epoch": 0.95, + "learning_rate": 3.133993080637665e-08, + "logits/chosen": -2.323228359222412, + "logits/rejected": -2.1346726417541504, + "logps/chosen": -214.2357177734375, + "logps/rejected": -221.05899047851562, + "loss": 1525.7791, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11354829370975494, + "rewards/margins": 0.1923418641090393, + "rewards/rejected": -0.30589017271995544, + "step": 14590 + }, + { + "epoch": 0.96, + "learning_rate": 3.0445094188342186e-08, + "logits/chosen": -2.137878894805908, + "logits/rejected": -1.8444684743881226, + "logps/chosen": -264.8699035644531, + "logps/rejected": -202.3433074951172, + "loss": 1412.3269, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11393339931964874, + "rewards/margins": 0.17737577855587006, + "rewards/rejected": -0.2913091778755188, + "step": 14600 + }, + { + "epoch": 0.96, + "eval_logits/chosen": -2.345289707183838, + "eval_logits/rejected": -2.158090829849243, + "eval_logps/chosen": -243.47486877441406, + "eval_logps/rejected": -238.74017333984375, + "eval_loss": 1756.218994140625, + "eval_rewards/accuracies": 0.6654999852180481, + "eval_rewards/chosen": -0.11469940841197968, + "eval_rewards/margins": 0.15658311545848846, + "eval_rewards/rejected": -0.27128249406814575, + "eval_runtime": 720.8174, + "eval_samples_per_second": 2.775, + "eval_steps_per_second": 1.387, + "step": 14600 + }, + { + "epoch": 0.96, + "learning_rate": 2.9563139978421028e-08, + "logits/chosen": -2.252467632293701, + "logits/rejected": -2.276352882385254, + "logps/chosen": -237.4102325439453, + "logps/rejected": -239.51889038085938, + "loss": 2279.3187, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11538355052471161, + "rewards/margins": 0.09893187135457993, + "rewards/rejected": -0.21431544423103333, + "step": 14610 + }, + { + "epoch": 0.96, + "learning_rate": 2.869407277731939e-08, + "logits/chosen": -2.2228925228118896, + "logits/rejected": -2.151127815246582, + "logps/chosen": -194.53271484375, + "logps/rejected": -187.36260986328125, + "loss": 1568.2383, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09275592863559723, + "rewards/margins": 0.15999475121498108, + "rewards/rejected": -0.2527506649494171, + "step": 14620 + }, + { + "epoch": 0.96, + "learning_rate": 2.783789711851642e-08, + "logits/chosen": -2.3487181663513184, + "logits/rejected": -2.1291961669921875, + "logps/chosen": -169.86410522460938, + "logps/rejected": -167.00466918945312, + "loss": 1597.5015, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.08013113588094711, + "rewards/margins": 0.1957322657108307, + "rewards/rejected": -0.275863379240036, + "step": 14630 + }, + { + "epoch": 0.96, + "learning_rate": 2.6994617468244778e-08, + "logits/chosen": -2.446563243865967, + "logits/rejected": -1.968849539756775, + "logps/chosen": -224.6818084716797, + "logps/rejected": -178.614013671875, + "loss": 1439.7357, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11661139875650406, + "rewards/margins": 0.1785092055797577, + "rewards/rejected": -0.29512059688568115, + "step": 14640 + }, + { + "epoch": 0.96, + "learning_rate": 2.6164238225463155e-08, + "logits/chosen": -2.2791390419006348, + "logits/rejected": -1.9296939373016357, + "logps/chosen": -291.14801025390625, + "logps/rejected": -232.16015625, + "loss": 1572.7487, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.0872085690498352, + "rewards/margins": 0.19520151615142822, + "rewards/rejected": -0.2824100852012634, + "step": 14650 + }, + { + "epoch": 0.96, + "learning_rate": 2.534676372183742e-08, + "logits/chosen": -2.3055472373962402, + "logits/rejected": -2.1529393196105957, + "logps/chosen": -293.67352294921875, + "logps/rejected": -260.25726318359375, + "loss": 1817.2176, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09719154983758926, + "rewards/margins": 0.1891038864850998, + "rewards/rejected": -0.28629544377326965, + "step": 14660 + }, + { + "epoch": 0.96, + "learning_rate": 2.4542198221714218e-08, + "logits/chosen": -2.205369710922241, + "logits/rejected": -1.9309327602386475, + "logps/chosen": -148.51953125, + "logps/rejected": -163.5492401123047, + "loss": 1798.3756, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1041257381439209, + "rewards/margins": 0.14802828431129456, + "rewards/rejected": -0.25215405225753784, + "step": 14670 + }, + { + "epoch": 0.96, + "learning_rate": 2.3750545922101854e-08, + "logits/chosen": -2.601306915283203, + "logits/rejected": -2.2248146533966064, + "logps/chosen": -320.0009460449219, + "logps/rejected": -269.42388916015625, + "loss": 1441.0441, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.10609765350818634, + "rewards/margins": 0.1781700849533081, + "rewards/rejected": -0.28426772356033325, + "step": 14680 + }, + { + "epoch": 0.96, + "learning_rate": 2.2971810952646112e-08, + "logits/chosen": -2.3170502185821533, + "logits/rejected": -2.2311322689056396, + "logps/chosen": -270.6275939941406, + "logps/rejected": -233.7604217529297, + "loss": 2079.441, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.12264804542064667, + "rewards/margins": 0.10838682949542999, + "rewards/rejected": -0.23103487491607666, + "step": 14690 + }, + { + "epoch": 0.96, + "learning_rate": 2.2205997375610576e-08, + "logits/chosen": -2.1689231395721436, + "logits/rejected": -2.1099231243133545, + "logps/chosen": -190.51622009277344, + "logps/rejected": -215.9954071044922, + "loss": 1504.9481, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.050102997571229935, + "rewards/margins": 0.20319123566150665, + "rewards/rejected": -0.2532942295074463, + "step": 14700 + }, + { + "epoch": 0.96, + "eval_logits/chosen": -2.3439536094665527, + "eval_logits/rejected": -2.156829833984375, + "eval_logps/chosen": -243.49813842773438, + "eval_logps/rejected": -238.74496459960938, + "eval_loss": 1756.396728515625, + "eval_rewards/accuracies": 0.6644999980926514, + "eval_rewards/chosen": -0.1149320900440216, + "eval_rewards/margins": 0.15639850497245789, + "eval_rewards/rejected": -0.2713305652141571, + "eval_runtime": 723.7336, + "eval_samples_per_second": 2.763, + "eval_steps_per_second": 1.382, + "step": 14700 + }, + { + "epoch": 0.96, + "learning_rate": 2.1453109185853304e-08, + "logits/chosen": -2.3741512298583984, + "logits/rejected": -2.3167805671691895, + "logps/chosen": -210.87539672851562, + "logps/rejected": -226.72695922851562, + "loss": 1983.609, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.08796913921833038, + "rewards/margins": 0.14104311168193817, + "rewards/rejected": -0.22901228070259094, + "step": 14710 + }, + { + "epoch": 0.96, + "learning_rate": 2.0713150310808784e-08, + "logits/chosen": -2.1360185146331787, + "logits/rejected": -2.4715590476989746, + "logps/chosen": -238.7700958251953, + "logps/rejected": -251.58615112304688, + "loss": 2161.6273, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.13282257318496704, + "rewards/margins": 0.07052706182003021, + "rewards/rejected": -0.20334962010383606, + "step": 14720 + }, + { + "epoch": 0.96, + "learning_rate": 1.9986124610464064e-08, + "logits/chosen": -2.2351553440093994, + "logits/rejected": -1.9309247732162476, + "logps/chosen": -301.1767578125, + "logps/rejected": -252.90139770507812, + "loss": 1462.9733, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.1394665241241455, + "rewards/margins": 0.2153692990541458, + "rewards/rejected": -0.3548358082771301, + "step": 14730 + }, + { + "epoch": 0.96, + "learning_rate": 1.927203587734211e-08, + "logits/chosen": -2.2187814712524414, + "logits/rejected": -1.8177343606948853, + "logps/chosen": -258.6115417480469, + "logps/rejected": -231.11123657226562, + "loss": 1640.377, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.05619867891073227, + "rewards/margins": 0.2102610170841217, + "rewards/rejected": -0.26645970344543457, + "step": 14740 + }, + { + "epoch": 0.97, + "learning_rate": 1.8570887836479034e-08, + "logits/chosen": -2.334831714630127, + "logits/rejected": -2.1394412517547607, + "logps/chosen": -209.2335968017578, + "logps/rejected": -275.2274475097656, + "loss": 1893.7287, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1561044305562973, + "rewards/margins": 0.1317848265171051, + "rewards/rejected": -0.2878892421722412, + "step": 14750 + }, + { + "epoch": 0.97, + "learning_rate": 1.7882684145406616e-08, + "logits/chosen": -2.3986196517944336, + "logits/rejected": -2.3380439281463623, + "logps/chosen": -304.5469055175781, + "logps/rejected": -319.450927734375, + "loss": 1727.3447, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08126304298639297, + "rewards/margins": 0.16792842745780945, + "rewards/rejected": -0.24919147789478302, + "step": 14760 + }, + { + "epoch": 0.97, + "learning_rate": 1.7207428394132865e-08, + "logits/chosen": -2.553154945373535, + "logits/rejected": -2.0709140300750732, + "logps/chosen": -279.75396728515625, + "logps/rejected": -248.90267944335938, + "loss": 1438.3188, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.09973666071891785, + "rewards/margins": 0.2383594959974289, + "rewards/rejected": -0.3380962014198303, + "step": 14770 + }, + { + "epoch": 0.97, + "learning_rate": 1.654512410512177e-08, + "logits/chosen": -2.277434825897217, + "logits/rejected": -2.038137912750244, + "logps/chosen": -265.58087158203125, + "logps/rejected": -216.32894897460938, + "loss": 1964.3506, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12003912776708603, + "rewards/margins": 0.13136547803878784, + "rewards/rejected": -0.25140461325645447, + "step": 14780 + }, + { + "epoch": 0.97, + "learning_rate": 1.5895774733277468e-08, + "logits/chosen": -2.3411812782287598, + "logits/rejected": -1.9985847473144531, + "logps/chosen": -289.6302490234375, + "logps/rejected": -257.2305603027344, + "loss": 1619.169, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09813287854194641, + "rewards/margins": 0.17836883664131165, + "rewards/rejected": -0.27650171518325806, + "step": 14790 + }, + { + "epoch": 0.97, + "learning_rate": 1.5259383665924e-08, + "logits/chosen": -2.5866286754608154, + "logits/rejected": -2.2018492221832275, + "logps/chosen": -348.4439392089844, + "logps/rejected": -275.0132751464844, + "loss": 1509.7718, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06352052092552185, + "rewards/margins": 0.20578232407569885, + "rewards/rejected": -0.2693028748035431, + "step": 14800 + }, + { + "epoch": 0.97, + "eval_logits/chosen": -2.344806432723999, + "eval_logits/rejected": -2.1576449871063232, + "eval_logps/chosen": -243.4784393310547, + "eval_logps/rejected": -238.7247772216797, + "eval_loss": 1755.3797607421875, + "eval_rewards/accuracies": 0.6625000238418579, + "eval_rewards/chosen": -0.11473479121923447, + "eval_rewards/margins": 0.15639401972293854, + "eval_rewards/rejected": -0.2711288034915924, + "eval_runtime": 719.5509, + "eval_samples_per_second": 2.78, + "eval_steps_per_second": 1.39, + "step": 14800 + }, + { + "epoch": 0.97, + "learning_rate": 1.4635954222789461e-08, + "logits/chosen": -2.2872562408447266, + "logits/rejected": -2.202056884765625, + "logps/chosen": -228.4236297607422, + "logps/rejected": -248.9585723876953, + "loss": 1902.5502, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1131603866815567, + "rewards/margins": 0.15427032113075256, + "rewards/rejected": -0.26743072271347046, + "step": 14810 + }, + { + "epoch": 0.97, + "learning_rate": 1.402548965598688e-08, + "logits/chosen": -2.222163438796997, + "logits/rejected": -2.327205181121826, + "logps/chosen": -216.5567626953125, + "logps/rejected": -231.3907012939453, + "loss": 1527.4884, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.11929510533809662, + "rewards/margins": 0.1805037558078766, + "rewards/rejected": -0.2997989058494568, + "step": 14820 + }, + { + "epoch": 0.97, + "learning_rate": 1.3427993149998375e-08, + "logits/chosen": -2.4768872261047363, + "logits/rejected": -2.2277302742004395, + "logps/chosen": -252.4502716064453, + "logps/rejected": -213.08779907226562, + "loss": 1706.2182, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.10413049161434174, + "rewards/margins": 0.16007684171199799, + "rewards/rejected": -0.2642073631286621, + "step": 14830 + }, + { + "epoch": 0.97, + "learning_rate": 1.2843467821658518e-08, + "logits/chosen": -2.433828830718994, + "logits/rejected": -2.370725154876709, + "logps/chosen": -230.56201171875, + "logps/rejected": -249.99472045898438, + "loss": 1390.466, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.04339112713932991, + "rewards/margins": 0.19801194965839386, + "rewards/rejected": -0.24140305817127228, + "step": 14840 + }, + { + "epoch": 0.97, + "learning_rate": 1.2271916720137666e-08, + "logits/chosen": -2.5363497734069824, + "logits/rejected": -2.2229340076446533, + "logps/chosen": -299.01824951171875, + "logps/rejected": -264.07366943359375, + "loss": 2100.2973, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.13248726725578308, + "rewards/margins": 0.11190402507781982, + "rewards/rejected": -0.2443912923336029, + "step": 14850 + }, + { + "epoch": 0.97, + "learning_rate": 1.171334282692671e-08, + "logits/chosen": -2.3759212493896484, + "logits/rejected": -2.312915563583374, + "logps/chosen": -290.940673828125, + "logps/rejected": -282.8968505859375, + "loss": 1562.8558, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.08342249691486359, + "rewards/margins": 0.22588083148002625, + "rewards/rejected": -0.30930331349372864, + "step": 14860 + }, + { + "epoch": 0.97, + "learning_rate": 1.116774905582041e-08, + "logits/chosen": -2.4384074211120605, + "logits/rejected": -2.107351779937744, + "logps/chosen": -196.92654418945312, + "logps/rejected": -195.10423278808594, + "loss": 1941.9195, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.12108032405376434, + "rewards/margins": 0.10880926996469498, + "rewards/rejected": -0.22988960146903992, + "step": 14870 + }, + { + "epoch": 0.97, + "learning_rate": 1.0635138252902966e-08, + "logits/chosen": -2.518921375274658, + "logits/rejected": -2.26084303855896, + "logps/chosen": -241.8000946044922, + "logps/rejected": -236.13381958007812, + "loss": 1661.2465, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11386130750179291, + "rewards/margins": 0.16462181508541107, + "rewards/rejected": -0.278483122587204, + "step": 14880 + }, + { + "epoch": 0.97, + "learning_rate": 1.0115513196533589e-08, + "logits/chosen": -2.3540704250335693, + "logits/rejected": -2.1872241497039795, + "logps/chosen": -273.9299011230469, + "logps/rejected": -268.36236572265625, + "loss": 2094.5197, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11319758743047714, + "rewards/margins": 0.11740664392709732, + "rewards/rejected": -0.23060421645641327, + "step": 14890 + }, + { + "epoch": 0.97, + "learning_rate": 9.608876597330952e-09, + "logits/chosen": -2.3617584705352783, + "logits/rejected": -2.0387043952941895, + "logps/chosen": -306.14923095703125, + "logps/rejected": -309.53643798828125, + "loss": 1881.0627, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.19526515901088715, + "rewards/margins": 0.1507243812084198, + "rewards/rejected": -0.34598949551582336, + "step": 14900 + }, + { + "epoch": 0.97, + "eval_logits/chosen": -2.3451874256134033, + "eval_logits/rejected": -2.158019781112671, + "eval_logps/chosen": -243.46603393554688, + "eval_logps/rejected": -238.71018981933594, + "eval_loss": 1755.147216796875, + "eval_rewards/accuracies": 0.6650000214576721, + "eval_rewards/chosen": -0.1146109476685524, + "eval_rewards/margins": 0.1563720554113388, + "eval_rewards/rejected": -0.2709830403327942, + "eval_runtime": 721.9629, + "eval_samples_per_second": 2.77, + "eval_steps_per_second": 1.385, + "step": 14900 + }, + { + "epoch": 0.98, + "learning_rate": 9.115231098159594e-09, + "logits/chosen": -2.4154810905456543, + "logits/rejected": -2.3058924674987793, + "logps/chosen": -263.7437438964844, + "logps/rejected": -255.2856903076172, + "loss": 2138.4381, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10429998487234116, + "rewards/margins": 0.11843911558389664, + "rewards/rejected": -0.22273913025856018, + "step": 14910 + }, + { + "epoch": 0.98, + "learning_rate": 8.634579274116317e-09, + "logits/chosen": -2.2822928428649902, + "logits/rejected": -2.213261127471924, + "logps/chosen": -199.53839111328125, + "logps/rejected": -233.07620239257812, + "loss": 2012.0393, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0947912260890007, + "rewards/margins": 0.12928275763988495, + "rewards/rejected": -0.22407397627830505, + "step": 14920 + }, + { + "epoch": 0.98, + "learning_rate": 8.166923632516865e-09, + "logits/chosen": -2.4574990272521973, + "logits/rejected": -2.1689932346343994, + "logps/chosen": -235.9760284423828, + "logps/rejected": -298.189208984375, + "loss": 1338.2018, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.08244818449020386, + "rewards/margins": 0.24586090445518494, + "rewards/rejected": -0.3283091187477112, + "step": 14930 + }, + { + "epoch": 0.98, + "learning_rate": 7.712266612881492e-09, + "logits/chosen": -2.2132229804992676, + "logits/rejected": -2.084080219268799, + "logps/chosen": -192.06051635742188, + "logps/rejected": -206.5467071533203, + "loss": 1505.5216, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.06823839992284775, + "rewards/margins": 0.1936199814081192, + "rewards/rejected": -0.26185840368270874, + "step": 14940 + }, + { + "epoch": 0.98, + "learning_rate": 7.270610586924687e-09, + "logits/chosen": -2.463947057723999, + "logits/rejected": -2.2258262634277344, + "logps/chosen": -273.31195068359375, + "logps/rejected": -246.1824493408203, + "loss": 1626.5279, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08346611261367798, + "rewards/margins": 0.16950808465480804, + "rewards/rejected": -0.2529742121696472, + "step": 14950 + }, + { + "epoch": 0.98, + "learning_rate": 6.841957858539916e-09, + "logits/chosen": -2.284597158432007, + "logits/rejected": -2.139622211456299, + "logps/chosen": -185.13320922851562, + "logps/rejected": -206.1970672607422, + "loss": 2098.2066, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.18477652966976166, + "rewards/margins": 0.10213065147399902, + "rewards/rejected": -0.2869071960449219, + "step": 14960 + }, + { + "epoch": 0.98, + "learning_rate": 6.426310663790181e-09, + "logits/chosen": -2.215423822402954, + "logits/rejected": -2.1268699169158936, + "logps/chosen": -252.93124389648438, + "logps/rejected": -235.5459442138672, + "loss": 2000.2178, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1595548391342163, + "rewards/margins": 0.14083118736743927, + "rewards/rejected": -0.3003860414028168, + "step": 14970 + }, + { + "epoch": 0.98, + "learning_rate": 6.023671170894696e-09, + "logits/chosen": -2.530273675918579, + "logits/rejected": -1.9427354335784912, + "logps/chosen": -300.4677429199219, + "logps/rejected": -238.73995971679688, + "loss": 1062.9697, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08139168471097946, + "rewards/margins": 0.25051823258399963, + "rewards/rejected": -0.3319099247455597, + "step": 14980 + }, + { + "epoch": 0.98, + "learning_rate": 5.634041480218344e-09, + "logits/chosen": -2.4931185245513916, + "logits/rejected": -2.1452651023864746, + "logps/chosen": -262.18927001953125, + "logps/rejected": -275.85614013671875, + "loss": 1671.9363, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.09115663915872574, + "rewards/margins": 0.18107278645038605, + "rewards/rejected": -0.27222940325737, + "step": 14990 + }, + { + "epoch": 0.98, + "learning_rate": 5.257423624260849e-09, + "logits/chosen": -2.5348258018493652, + "logits/rejected": -2.100780487060547, + "logps/chosen": -273.679443359375, + "logps/rejected": -246.7283477783203, + "loss": 1820.8113, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10356035083532333, + "rewards/margins": 0.16292151808738708, + "rewards/rejected": -0.266481876373291, + "step": 15000 + }, + { + "epoch": 0.98, + "eval_logits/chosen": -2.3453927040100098, + "eval_logits/rejected": -2.1581103801727295, + "eval_logps/chosen": -243.4661102294922, + "eval_logps/rejected": -238.70730590820312, + "eval_loss": 1754.8675537109375, + "eval_rewards/accuracies": 0.6660000085830688, + "eval_rewards/chosen": -0.11461148411035538, + "eval_rewards/margins": 0.1563425213098526, + "eval_rewards/rejected": -0.2709540128707886, + "eval_runtime": 720.1314, + "eval_samples_per_second": 2.777, + "eval_steps_per_second": 1.389, + "step": 15000 + }, + { + "epoch": 0.98, + "learning_rate": 4.893819567644564e-09, + "logits/chosen": -2.2239840030670166, + "logits/rejected": -2.200934410095215, + "logps/chosen": -198.62867736816406, + "logps/rejected": -220.85385131835938, + "loss": 2058.44, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10631934553384781, + "rewards/margins": 0.1204916462302208, + "rewards/rejected": -0.2268110066652298, + "step": 15010 + }, + { + "epoch": 0.98, + "learning_rate": 4.543231207107257e-09, + "logits/chosen": -2.2935476303100586, + "logits/rejected": -2.0775296688079834, + "logps/chosen": -268.8769226074219, + "logps/rejected": -260.10272216796875, + "loss": 1846.2023, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11055328696966171, + "rewards/margins": 0.14834541082382202, + "rewards/rejected": -0.25889870524406433, + "step": 15020 + }, + { + "epoch": 0.98, + "learning_rate": 4.205660371488785e-09, + "logits/chosen": -2.585259437561035, + "logits/rejected": -2.224368095397949, + "logps/chosen": -295.079345703125, + "logps/rejected": -267.5276794433594, + "loss": 2099.5447, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10925974696874619, + "rewards/margins": 0.14238341152668, + "rewards/rejected": -0.2516431510448456, + "step": 15030 + }, + { + "epoch": 0.98, + "learning_rate": 3.88110882172471e-09, + "logits/chosen": -2.2619118690490723, + "logits/rejected": -2.2005677223205566, + "logps/chosen": -234.2068634033203, + "logps/rejected": -243.8390350341797, + "loss": 1841.5732, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1350843608379364, + "rewards/margins": 0.1426878273487091, + "rewards/rejected": -0.2777721583843231, + "step": 15040 + }, + { + "epoch": 0.98, + "learning_rate": 3.569578250834371e-09, + "logits/chosen": -2.363293170928955, + "logits/rejected": -2.089813709259033, + "logps/chosen": -308.189208984375, + "logps/rejected": -291.08929443359375, + "loss": 1346.5727, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06211404129862785, + "rewards/margins": 0.20662875473499298, + "rewards/rejected": -0.26874274015426636, + "step": 15050 + }, + { + "epoch": 0.99, + "learning_rate": 3.2710702839139353e-09, + "logits/chosen": -2.390526533126831, + "logits/rejected": -2.2443912029266357, + "logps/chosen": -214.2931671142578, + "logps/rejected": -234.26174926757812, + "loss": 1999.3119, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0969388335943222, + "rewards/margins": 0.10695371776819229, + "rewards/rejected": -0.2038925439119339, + "step": 15060 + }, + { + "epoch": 0.99, + "learning_rate": 2.9855864781272448e-09, + "logits/chosen": -2.3530771732330322, + "logits/rejected": -2.389768600463867, + "logps/chosen": -219.4556884765625, + "logps/rejected": -269.90338134765625, + "loss": 1778.8965, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07782004773616791, + "rewards/margins": 0.15653666853904724, + "rewards/rejected": -0.23435673117637634, + "step": 15070 + }, + { + "epoch": 0.99, + "learning_rate": 2.7131283226977665e-09, + "logits/chosen": -2.359271287918091, + "logits/rejected": -2.429197311401367, + "logps/chosen": -229.75521850585938, + "logps/rejected": -259.5672607421875, + "loss": 1777.0398, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.10306936502456665, + "rewards/margins": 0.15961900353431702, + "rewards/rejected": -0.26268836855888367, + "step": 15080 + }, + { + "epoch": 0.99, + "learning_rate": 2.4536972389008205e-09, + "logits/chosen": -2.309981346130371, + "logits/rejected": -2.077146053314209, + "logps/chosen": -246.08938598632812, + "logps/rejected": -227.2568359375, + "loss": 1641.2615, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08954377472400665, + "rewards/margins": 0.18858453631401062, + "rewards/rejected": -0.27812832593917847, + "step": 15090 + }, + { + "epoch": 0.99, + "learning_rate": 2.20729458005553e-09, + "logits/chosen": -2.221825122833252, + "logits/rejected": -2.062643527984619, + "logps/chosen": -205.6139678955078, + "logps/rejected": -202.2505340576172, + "loss": 1512.1538, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11204735189676285, + "rewards/margins": 0.18838240206241608, + "rewards/rejected": -0.3004297614097595, + "step": 15100 + }, + { + "epoch": 0.99, + "eval_logits/chosen": -2.3453314304351807, + "eval_logits/rejected": -2.1580898761749268, + "eval_logps/chosen": -243.4615020751953, + "eval_logps/rejected": -238.70053100585938, + "eval_loss": 1754.2144775390625, + "eval_rewards/accuracies": 0.6635000109672546, + "eval_rewards/chosen": -0.11456552892923355, + "eval_rewards/margins": 0.15632058680057526, + "eval_rewards/rejected": -0.2708861231803894, + "eval_runtime": 720.0327, + "eval_samples_per_second": 2.778, + "eval_steps_per_second": 1.389, + "step": 15100 + }, + { + "epoch": 0.99, + "learning_rate": 1.9739216315192712e-09, + "logits/chosen": -2.3239998817443848, + "logits/rejected": -2.11338472366333, + "logps/chosen": -240.9944610595703, + "logps/rejected": -230.3395538330078, + "loss": 1798.9738, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11685649305582047, + "rewards/margins": 0.13042503595352173, + "rewards/rejected": -0.2472815066576004, + "step": 15110 + }, + { + "epoch": 0.99, + "learning_rate": 1.7535796106796231e-09, + "logits/chosen": -2.4070184230804443, + "logits/rejected": -2.0923027992248535, + "logps/chosen": -286.97772216796875, + "logps/rejected": -220.5601348876953, + "loss": 1745.35, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11441268771886826, + "rewards/margins": 0.13913391530513763, + "rewards/rejected": -0.25354665517807007, + "step": 15120 + }, + { + "epoch": 0.99, + "learning_rate": 1.5462696669482636e-09, + "logits/chosen": -2.376131534576416, + "logits/rejected": -2.258821487426758, + "logps/chosen": -232.0293426513672, + "logps/rejected": -252.01962280273438, + "loss": 1955.3076, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10787205398082733, + "rewards/margins": 0.12346227467060089, + "rewards/rejected": -0.23133432865142822, + "step": 15130 + }, + { + "epoch": 0.99, + "learning_rate": 1.3519928817556927e-09, + "logits/chosen": -2.232820987701416, + "logits/rejected": -2.192532777786255, + "logps/chosen": -183.15576171875, + "logps/rejected": -196.3177032470703, + "loss": 1906.4008, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09762244671583176, + "rewards/margins": 0.12615005671977997, + "rewards/rejected": -0.22377249598503113, + "step": 15140 + }, + { + "epoch": 0.99, + "learning_rate": 1.1707502685448512e-09, + "logits/chosen": -2.4631078243255615, + "logits/rejected": -2.126909017562866, + "logps/chosen": -240.1180419921875, + "logps/rejected": -200.20216369628906, + "loss": 1534.8595, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1784803569316864, + "rewards/margins": 0.18272601068019867, + "rewards/rejected": -0.3612063527107239, + "step": 15150 + }, + { + "epoch": 0.99, + "learning_rate": 1.002542772765569e-09, + "logits/chosen": -2.2957310676574707, + "logits/rejected": -2.0127415657043457, + "logps/chosen": -198.36758422851562, + "logps/rejected": -177.6871337890625, + "loss": 1611.424, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.0976375862956047, + "rewards/margins": 0.1762719601392746, + "rewards/rejected": -0.2739095389842987, + "step": 15160 + }, + { + "epoch": 0.99, + "learning_rate": 8.473712718709559e-10, + "logits/chosen": -2.1500306129455566, + "logits/rejected": -2.168015956878662, + "logps/chosen": -209.2052764892578, + "logps/rejected": -206.0652618408203, + "loss": 2103.8172, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1546037644147873, + "rewards/margins": 0.07638068497180939, + "rewards/rejected": -0.23098444938659668, + "step": 15170 + }, + { + "epoch": 0.99, + "learning_rate": 7.052365753112966e-10, + "logits/chosen": -2.1892058849334717, + "logits/rejected": -1.8921606540679932, + "logps/chosen": -250.9332275390625, + "logps/rejected": -247.93362426757812, + "loss": 1891.3033, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.15617124736309052, + "rewards/margins": 0.15776070952415466, + "rewards/rejected": -0.31393197178840637, + "step": 15180 + }, + { + "epoch": 0.99, + "learning_rate": 5.761394245307195e-10, + "logits/chosen": -2.1700949668884277, + "logits/rejected": -2.1966946125030518, + "logps/chosen": -247.8317108154297, + "logps/rejected": -257.30291748046875, + "loss": 1964.2982, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10749546438455582, + "rewards/margins": 0.10682658851146698, + "rewards/rejected": -0.2143220603466034, + "step": 15190 + }, + { + "epoch": 0.99, + "learning_rate": 4.6008049296358826e-10, + "logits/chosen": -2.253401041030884, + "logits/rejected": -2.1524932384490967, + "logps/chosen": -196.65591430664062, + "logps/rejected": -184.73748779296875, + "loss": 1698.8312, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10633338987827301, + "rewards/margins": 0.16325244307518005, + "rewards/rejected": -0.26958581805229187, + "step": 15200 + }, + { + "epoch": 0.99, + "eval_logits/chosen": -2.344879388809204, + "eval_logits/rejected": -2.1577203273773193, + "eval_logps/chosen": -243.4614715576172, + "eval_logps/rejected": -238.69151306152344, + "eval_loss": 1755.1842041015625, + "eval_rewards/accuracies": 0.6629999876022339, + "eval_rewards/chosen": -0.11456498503684998, + "eval_rewards/margins": 0.15623119473457336, + "eval_rewards/rejected": -0.27079617977142334, + "eval_runtime": 718.3605, + "eval_samples_per_second": 2.784, + "eval_steps_per_second": 1.392, + "step": 15200 + }, + { + "epoch": 1.0, + "learning_rate": 3.5706038603006146e-10, + "logits/chosen": -2.430497646331787, + "logits/rejected": -2.3825058937072754, + "logps/chosen": -290.90673828125, + "logps/rejected": -296.6795654296875, + "loss": 1553.6001, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.012963606044650078, + "rewards/margins": 0.20015950500965118, + "rewards/rejected": -0.21312308311462402, + "step": 15210 + }, + { + "epoch": 1.0, + "learning_rate": 2.670796411333165e-10, + "logits/chosen": -2.608908176422119, + "logits/rejected": -2.3190643787384033, + "logps/chosen": -232.03125, + "logps/rejected": -234.5654754638672, + "loss": 1794.9951, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12219800800085068, + "rewards/margins": 0.14840340614318848, + "rewards/rejected": -0.27060142159461975, + "step": 15220 + }, + { + "epoch": 1.0, + "learning_rate": 1.9013872765677455e-10, + "logits/chosen": -2.3324027061462402, + "logits/rejected": -2.127634048461914, + "logps/chosen": -227.2728271484375, + "logps/rejected": -227.78005981445312, + "loss": 1788.8002, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.07759709656238556, + "rewards/margins": 0.13041789829730988, + "rewards/rejected": -0.20801499485969543, + "step": 15230 + }, + { + "epoch": 1.0, + "learning_rate": 1.262380469624347e-10, + "logits/chosen": -2.310046672821045, + "logits/rejected": -2.1428942680358887, + "logps/chosen": -214.257080078125, + "logps/rejected": -206.60110473632812, + "loss": 1765.4699, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.10250154882669449, + "rewards/margins": 0.15253737568855286, + "rewards/rejected": -0.25503891706466675, + "step": 15240 + }, + { + "epoch": 1.0, + "learning_rate": 7.53779323872661e-11, + "logits/chosen": -2.2056996822357178, + "logits/rejected": -2.3175556659698486, + "logps/chosen": -202.9796142578125, + "logps/rejected": -226.6588134765625, + "loss": 2022.0285, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1111622080206871, + "rewards/margins": 0.17635276913642883, + "rewards/rejected": -0.28751498460769653, + "step": 15250 + }, + { + "epoch": 1.0, + "learning_rate": 3.7558649242652734e-11, + "logits/chosen": -2.5009121894836426, + "logits/rejected": -2.27042818069458, + "logps/chosen": -406.96636962890625, + "logps/rejected": -343.4335021972656, + "loss": 1724.1916, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.10282541811466217, + "rewards/margins": 0.16037984192371368, + "rewards/rejected": -0.26320526003837585, + "step": 15260 + }, + { + "epoch": 1.0, + "learning_rate": 1.2780394812450526e-11, + "logits/chosen": -2.1512584686279297, + "logits/rejected": -2.1113929748535156, + "logps/chosen": -247.2060546875, + "logps/rejected": -264.48095703125, + "loss": 1595.7446, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15062271058559418, + "rewards/margins": 0.17357513308525085, + "rewards/rejected": -0.3241978883743286, + "step": 15270 + }, + { + "epoch": 1.0, + "learning_rate": 1.0432983521546646e-12, + "logits/chosen": -2.191702365875244, + "logits/rejected": -2.101569890975952, + "logps/chosen": -194.95314025878906, + "logps/rejected": -227.9114227294922, + "loss": 1693.5664, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1148693710565567, + "rewards/margins": 0.1411261111497879, + "rewards/rejected": -0.2559954822063446, + "step": 15280 + }, + { + "epoch": 1.0, + "step": 15284, + "total_flos": 0.0, + "train_loss": 1830.215161976219, + "train_runtime": 182270.758, + "train_samples_per_second": 0.335, + "train_steps_per_second": 0.084 + } + ], + "logging_steps": 10, + "max_steps": 15284, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 100, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}