diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,10772 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 3000, + "global_step": 7642, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 6.535947712418301e-09, + "logits/chosen": -3.15731143951416, + "logits/rejected": -2.798032760620117, + "logps/chosen": -362.5858154296875, + "logps/rejected": -239.5453338623047, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.0, + "learning_rate": 6.535947712418302e-08, + "logits/chosen": -2.847041130065918, + "logits/rejected": -2.83606219291687, + "logps/chosen": -298.7374267578125, + "logps/rejected": -247.803955078125, + "loss": 0.6925, + "rewards/accuracies": 0.5277777910232544, + "rewards/chosen": 0.0011123637668788433, + "rewards/margins": 0.0014208466745913029, + "rewards/rejected": -0.00030848290771245956, + "step": 10 + }, + { + "epoch": 0.0, + "learning_rate": 1.3071895424836603e-07, + "logits/chosen": -2.7611355781555176, + "logits/rejected": -2.752760648727417, + "logps/chosen": -262.10382080078125, + "logps/rejected": -281.1115417480469, + "loss": 0.6929, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.0011931858025491238, + "rewards/margins": 9.312294650953845e-07, + "rewards/rejected": 0.0011922543635591865, + "step": 20 + }, + { + "epoch": 0.0, + "learning_rate": 1.9607843137254904e-07, + "logits/chosen": -2.932368516921997, + "logits/rejected": -2.9066896438598633, + "logps/chosen": -252.71548461914062, + "logps/rejected": -236.00338745117188, + "loss": 0.6932, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.00011899097444256768, + "rewards/margins": 0.0004326296038925648, + "rewards/rejected": -0.00031363870948553085, + "step": 30 + }, + { + "epoch": 0.01, + "learning_rate": 2.6143790849673207e-07, + "logits/chosen": -2.873145341873169, + "logits/rejected": -2.880809783935547, + "logps/chosen": -259.3544921875, + "logps/rejected": -247.77493286132812, + "loss": 0.6929, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.0008003931725397706, + "rewards/margins": 0.0007967362180352211, + "rewards/rejected": 3.6569545045495033e-06, + "step": 40 + }, + { + "epoch": 0.01, + "learning_rate": 3.267973856209151e-07, + "logits/chosen": -2.883204460144043, + "logits/rejected": -2.9383983612060547, + "logps/chosen": -307.0992431640625, + "logps/rejected": -261.85723876953125, + "loss": 0.6925, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.0009268176509067416, + "rewards/margins": 0.0016272146021947265, + "rewards/rejected": -0.0007003970677033067, + "step": 50 + }, + { + "epoch": 0.01, + "learning_rate": 3.921568627450981e-07, + "logits/chosen": -2.952158212661743, + "logits/rejected": -2.9316773414611816, + "logps/chosen": -337.2663269042969, + "logps/rejected": -268.79132080078125, + "loss": 0.6934, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.0011328494874760509, + "rewards/margins": 0.000162464493769221, + "rewards/rejected": 0.0009703851537778974, + "step": 60 + }, + { + "epoch": 0.01, + "learning_rate": 4.5751633986928105e-07, + "logits/chosen": -2.8809123039245605, + "logits/rejected": -2.8493149280548096, + "logps/chosen": -296.692138671875, + "logps/rejected": -298.32220458984375, + "loss": 0.6932, + "rewards/accuracies": 0.3499999940395355, + "rewards/chosen": -0.00022940686903893948, + "rewards/margins": -0.00013572932220995426, + "rewards/rejected": -9.367751044919714e-05, + "step": 70 + }, + { + "epoch": 0.01, + "learning_rate": 5.228758169934641e-07, + "logits/chosen": -2.8788790702819824, + "logits/rejected": -2.8450794219970703, + "logps/chosen": -297.30792236328125, + "logps/rejected": -261.411376953125, + "loss": 0.6932, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.0009753646445460618, + "rewards/margins": 0.0006360862171277404, + "rewards/rejected": 0.00033927842741832137, + "step": 80 + }, + { + "epoch": 0.01, + "learning_rate": 5.882352941176471e-07, + "logits/chosen": -2.877856731414795, + "logits/rejected": -2.8819994926452637, + "logps/chosen": -253.0044708251953, + "logps/rejected": -222.84390258789062, + "loss": 0.6929, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.0003014683607034385, + "rewards/margins": 0.0006676745833829045, + "rewards/rejected": -0.0003662061644718051, + "step": 90 + }, + { + "epoch": 0.01, + "learning_rate": 6.535947712418302e-07, + "logits/chosen": -2.8481335639953613, + "logits/rejected": -2.7708733081817627, + "logps/chosen": -276.61749267578125, + "logps/rejected": -276.0560607910156, + "loss": 0.6936, + "rewards/accuracies": 0.375, + "rewards/chosen": 6.637955084443092e-05, + "rewards/margins": -0.0015732726315036416, + "rewards/rejected": 0.0016396520659327507, + "step": 100 + }, + { + "epoch": 0.01, + "learning_rate": 7.189542483660131e-07, + "logits/chosen": -2.812410831451416, + "logits/rejected": -2.8500609397888184, + "logps/chosen": -302.9320373535156, + "logps/rejected": -320.2331237792969, + "loss": 0.6933, + "rewards/accuracies": 0.375, + "rewards/chosen": 0.00024477095576003194, + "rewards/margins": -0.0008945292793214321, + "rewards/rejected": 0.0011393001768738031, + "step": 110 + }, + { + "epoch": 0.02, + "learning_rate": 7.843137254901962e-07, + "logits/chosen": -2.7782273292541504, + "logits/rejected": -2.786151170730591, + "logps/chosen": -235.53518676757812, + "logps/rejected": -234.79306030273438, + "loss": 0.6934, + "rewards/accuracies": 0.42500001192092896, + "rewards/chosen": 0.00024773445329628885, + "rewards/margins": -0.001447710208594799, + "rewards/rejected": 0.0016954450402408838, + "step": 120 + }, + { + "epoch": 0.02, + "learning_rate": 8.496732026143792e-07, + "logits/chosen": -2.6703851222991943, + "logits/rejected": -2.723660469055176, + "logps/chosen": -279.0316467285156, + "logps/rejected": -230.14675903320312, + "loss": 0.693, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0005290083936415613, + "rewards/margins": 0.0002718138857744634, + "rewards/rejected": 0.0002571945369709283, + "step": 130 + }, + { + "epoch": 0.02, + "learning_rate": 9.150326797385621e-07, + "logits/chosen": -2.8054141998291016, + "logits/rejected": -2.8203861713409424, + "logps/chosen": -239.762939453125, + "logps/rejected": -234.76779174804688, + "loss": 0.6931, + "rewards/accuracies": 0.5, + "rewards/chosen": -9.767948358785361e-05, + "rewards/margins": -0.000681340170558542, + "rewards/rejected": 0.0005836606142111123, + "step": 140 + }, + { + "epoch": 0.02, + "learning_rate": 9.80392156862745e-07, + "logits/chosen": -2.7740328311920166, + "logits/rejected": -2.7924256324768066, + "logps/chosen": -246.41683959960938, + "logps/rejected": -292.7257385253906, + "loss": 0.6934, + "rewards/accuracies": 0.32499998807907104, + "rewards/chosen": 9.760662214830518e-05, + "rewards/margins": -0.0012625637464225292, + "rewards/rejected": 0.0013601703103631735, + "step": 150 + }, + { + "epoch": 0.02, + "learning_rate": 1.0457516339869283e-06, + "logits/chosen": -2.7917070388793945, + "logits/rejected": -2.72552490234375, + "logps/chosen": -247.941650390625, + "logps/rejected": -256.9376220703125, + "loss": 0.6931, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.0016826872015371919, + "rewards/margins": 0.0002282693749293685, + "rewards/rejected": 0.0014544179430231452, + "step": 160 + }, + { + "epoch": 0.02, + "learning_rate": 1.111111111111111e-06, + "logits/chosen": -2.9668118953704834, + "logits/rejected": -2.855332612991333, + "logps/chosen": -272.6416320800781, + "logps/rejected": -224.99267578125, + "loss": 0.6932, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.0012751845642924309, + "rewards/margins": 0.0004584183043334633, + "rewards/rejected": 0.0008167661726474762, + "step": 170 + }, + { + "epoch": 0.02, + "learning_rate": 1.1764705882352942e-06, + "logits/chosen": -2.8704068660736084, + "logits/rejected": -2.86428165435791, + "logps/chosen": -198.56057739257812, + "logps/rejected": -208.8024444580078, + "loss": 0.6931, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0017385890241712332, + "rewards/margins": 0.0009926703060045838, + "rewards/rejected": 0.0007459186599589884, + "step": 180 + }, + { + "epoch": 0.02, + "learning_rate": 1.2418300653594772e-06, + "logits/chosen": -2.9300076961517334, + "logits/rejected": -2.964507818222046, + "logps/chosen": -298.9310302734375, + "logps/rejected": -239.85885620117188, + "loss": 0.6929, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": 0.0012464747997000813, + "rewards/margins": -0.000528771779499948, + "rewards/rejected": 0.0017752466956153512, + "step": 190 + }, + { + "epoch": 0.03, + "learning_rate": 1.3071895424836604e-06, + "logits/chosen": -2.8765249252319336, + "logits/rejected": -2.7931816577911377, + "logps/chosen": -252.77047729492188, + "logps/rejected": -220.84866333007812, + "loss": 0.6928, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.0029885726980865, + "rewards/margins": 0.0007496483740396798, + "rewards/rejected": 0.002238924615085125, + "step": 200 + }, + { + "epoch": 0.03, + "learning_rate": 1.3725490196078434e-06, + "logits/chosen": -2.8990020751953125, + "logits/rejected": -2.867999792098999, + "logps/chosen": -246.72994995117188, + "logps/rejected": -271.1702880859375, + "loss": 0.6928, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.0023519580718129873, + "rewards/margins": 0.001143458066508174, + "rewards/rejected": 0.0012085003545507789, + "step": 210 + }, + { + "epoch": 0.03, + "learning_rate": 1.4379084967320261e-06, + "logits/chosen": -2.86352276802063, + "logits/rejected": -2.8969032764434814, + "logps/chosen": -241.1385040283203, + "logps/rejected": -232.1473388671875, + "loss": 0.6929, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.001888754777610302, + "rewards/margins": -0.0003588128020055592, + "rewards/rejected": 0.0022475675214082003, + "step": 220 + }, + { + "epoch": 0.03, + "learning_rate": 1.5032679738562091e-06, + "logits/chosen": -2.8990299701690674, + "logits/rejected": -2.8520991802215576, + "logps/chosen": -261.7444152832031, + "logps/rejected": -233.3668670654297, + "loss": 0.6925, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.002989122411236167, + "rewards/margins": 0.0005400863010436296, + "rewards/rejected": 0.002449036343023181, + "step": 230 + }, + { + "epoch": 0.03, + "learning_rate": 1.5686274509803923e-06, + "logits/chosen": -2.907646656036377, + "logits/rejected": -2.817707061767578, + "logps/chosen": -282.6989440917969, + "logps/rejected": -309.9423828125, + "loss": 0.6925, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0033154035918414593, + "rewards/margins": 0.0005889445310458541, + "rewards/rejected": 0.002726459177210927, + "step": 240 + }, + { + "epoch": 0.03, + "learning_rate": 1.6339869281045753e-06, + "logits/chosen": -2.7770705223083496, + "logits/rejected": -2.824580669403076, + "logps/chosen": -272.4206848144531, + "logps/rejected": -261.48199462890625, + "loss": 0.6925, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.004111588001251221, + "rewards/margins": 0.001057318178936839, + "rewards/rejected": 0.0030542700551450253, + "step": 250 + }, + { + "epoch": 0.03, + "learning_rate": 1.6993464052287585e-06, + "logits/chosen": -2.868993043899536, + "logits/rejected": -2.860084056854248, + "logps/chosen": -339.67071533203125, + "logps/rejected": -264.09552001953125, + "loss": 0.6927, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.0046674818731844425, + "rewards/margins": 0.0013502954971045256, + "rewards/rejected": 0.0033171861432492733, + "step": 260 + }, + { + "epoch": 0.04, + "learning_rate": 1.7647058823529414e-06, + "logits/chosen": -2.97316575050354, + "logits/rejected": -2.9198946952819824, + "logps/chosen": -300.5587463378906, + "logps/rejected": -270.83416748046875, + "loss": 0.6927, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.004219466354697943, + "rewards/margins": 0.0005742977955378592, + "rewards/rejected": 0.0036451686173677444, + "step": 270 + }, + { + "epoch": 0.04, + "learning_rate": 1.8300653594771242e-06, + "logits/chosen": -2.8331806659698486, + "logits/rejected": -2.8653924465179443, + "logps/chosen": -303.3408508300781, + "logps/rejected": -245.0085906982422, + "loss": 0.6915, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.004968521185219288, + "rewards/margins": 0.0015199267072603106, + "rewards/rejected": 0.0034485943615436554, + "step": 280 + }, + { + "epoch": 0.04, + "learning_rate": 1.8954248366013072e-06, + "logits/chosen": -2.8719141483306885, + "logits/rejected": -2.8208324909210205, + "logps/chosen": -257.5692443847656, + "logps/rejected": -274.0968017578125, + "loss": 0.6931, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": 0.004160556010901928, + "rewards/margins": -0.0008697215234860778, + "rewards/rejected": 0.0050302776508033276, + "step": 290 + }, + { + "epoch": 0.04, + "learning_rate": 1.96078431372549e-06, + "logits/chosen": -2.9372448921203613, + "logits/rejected": -2.9619555473327637, + "logps/chosen": -271.332275390625, + "logps/rejected": -278.2612609863281, + "loss": 0.6916, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.008250665850937366, + "rewards/margins": 0.0011537367245182395, + "rewards/rejected": 0.007096928544342518, + "step": 300 + }, + { + "epoch": 0.04, + "learning_rate": 2.0261437908496734e-06, + "logits/chosen": -2.7562594413757324, + "logits/rejected": -2.7253847122192383, + "logps/chosen": -296.3819274902344, + "logps/rejected": -246.0597381591797, + "loss": 0.6918, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.008744063787162304, + "rewards/margins": 0.0032774447463452816, + "rewards/rejected": 0.005466618575155735, + "step": 310 + }, + { + "epoch": 0.04, + "learning_rate": 2.0915032679738565e-06, + "logits/chosen": -2.858609676361084, + "logits/rejected": -2.918300151824951, + "logps/chosen": -207.58529663085938, + "logps/rejected": -209.2336883544922, + "loss": 0.6913, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.011316259391605854, + "rewards/margins": 0.00452456995844841, + "rewards/rejected": 0.006791689433157444, + "step": 320 + }, + { + "epoch": 0.04, + "learning_rate": 2.1568627450980393e-06, + "logits/chosen": -3.0018887519836426, + "logits/rejected": -2.9340403079986572, + "logps/chosen": -324.8250427246094, + "logps/rejected": -265.0435485839844, + "loss": 0.6918, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.012073887512087822, + "rewards/margins": 0.004768506623804569, + "rewards/rejected": 0.007305382285267115, + "step": 330 + }, + { + "epoch": 0.04, + "learning_rate": 2.222222222222222e-06, + "logits/chosen": -2.821833372116089, + "logits/rejected": -2.827418088912964, + "logps/chosen": -325.23211669921875, + "logps/rejected": -223.10629272460938, + "loss": 0.6914, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.011198493652045727, + "rewards/margins": 0.004123987164348364, + "rewards/rejected": 0.007074506487697363, + "step": 340 + }, + { + "epoch": 0.05, + "learning_rate": 2.2875816993464053e-06, + "logits/chosen": -2.8357648849487305, + "logits/rejected": -2.8311212062835693, + "logps/chosen": -237.04891967773438, + "logps/rejected": -237.57809448242188, + "loss": 0.6907, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.012098469771444798, + "rewards/margins": 0.0031062206253409386, + "rewards/rejected": 0.008992249146103859, + "step": 350 + }, + { + "epoch": 0.05, + "learning_rate": 2.3529411764705885e-06, + "logits/chosen": -2.8127646446228027, + "logits/rejected": -2.7550415992736816, + "logps/chosen": -274.84478759765625, + "logps/rejected": -223.2805938720703, + "loss": 0.6911, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.016145002096891403, + "rewards/margins": 0.0037769395858049393, + "rewards/rejected": 0.012368063442409039, + "step": 360 + }, + { + "epoch": 0.05, + "learning_rate": 2.4183006535947716e-06, + "logits/chosen": -2.92099666595459, + "logits/rejected": -2.9377002716064453, + "logps/chosen": -241.8701629638672, + "logps/rejected": -393.9468078613281, + "loss": 0.6906, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.013012359850108624, + "rewards/margins": 0.0014860105002298951, + "rewards/rejected": 0.011526349931955338, + "step": 370 + }, + { + "epoch": 0.05, + "learning_rate": 2.4836601307189544e-06, + "logits/chosen": -2.8547887802124023, + "logits/rejected": -2.816089391708374, + "logps/chosen": -226.1658172607422, + "logps/rejected": -205.92953491210938, + "loss": 0.6902, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.01577352173626423, + "rewards/margins": 0.006564581301063299, + "rewards/rejected": 0.009208940900862217, + "step": 380 + }, + { + "epoch": 0.05, + "learning_rate": 2.549019607843137e-06, + "logits/chosen": -2.8543856143951416, + "logits/rejected": -2.809668779373169, + "logps/chosen": -304.81158447265625, + "logps/rejected": -267.87872314453125, + "loss": 0.6893, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": 0.01894640363752842, + "rewards/margins": 0.008028329350054264, + "rewards/rejected": 0.01091807335615158, + "step": 390 + }, + { + "epoch": 0.05, + "learning_rate": 2.6143790849673208e-06, + "logits/chosen": -2.906557083129883, + "logits/rejected": -2.9965012073516846, + "logps/chosen": -292.5071105957031, + "logps/rejected": -282.63116455078125, + "loss": 0.6903, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": 0.022404711693525314, + "rewards/margins": 0.005580331198871136, + "rewards/rejected": 0.016824383288621902, + "step": 400 + }, + { + "epoch": 0.05, + "learning_rate": 2.6797385620915036e-06, + "logits/chosen": -2.950514793395996, + "logits/rejected": -2.924778461456299, + "logps/chosen": -254.015869140625, + "logps/rejected": -235.6728973388672, + "loss": 0.6914, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.019588075578212738, + "rewards/margins": 0.006429559551179409, + "rewards/rejected": 0.013158516958355904, + "step": 410 + }, + { + "epoch": 0.05, + "learning_rate": 2.7450980392156867e-06, + "logits/chosen": -2.8651256561279297, + "logits/rejected": -2.859278917312622, + "logps/chosen": -397.34173583984375, + "logps/rejected": -366.5126953125, + "loss": 0.691, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.023986633867025375, + "rewards/margins": 0.005491149611771107, + "rewards/rejected": 0.018495485186576843, + "step": 420 + }, + { + "epoch": 0.06, + "learning_rate": 2.8104575163398695e-06, + "logits/chosen": -2.8140807151794434, + "logits/rejected": -2.8027613162994385, + "logps/chosen": -284.19488525390625, + "logps/rejected": -286.6159362792969, + "loss": 0.6877, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.024940496310591698, + "rewards/margins": 0.010390857234597206, + "rewards/rejected": 0.014549640007317066, + "step": 430 + }, + { + "epoch": 0.06, + "learning_rate": 2.8758169934640523e-06, + "logits/chosen": -2.91127872467041, + "logits/rejected": -2.9656119346618652, + "logps/chosen": -289.0911865234375, + "logps/rejected": -265.87554931640625, + "loss": 0.6887, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.02747970260679722, + "rewards/margins": 0.00899188220500946, + "rewards/rejected": 0.018487820401787758, + "step": 440 + }, + { + "epoch": 0.06, + "learning_rate": 2.9411764705882355e-06, + "logits/chosen": -2.8094940185546875, + "logits/rejected": -2.7788679599761963, + "logps/chosen": -234.64382934570312, + "logps/rejected": -245.06405639648438, + "loss": 0.6896, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.02517036534845829, + "rewards/margins": 0.008376935496926308, + "rewards/rejected": 0.01679343171417713, + "step": 450 + }, + { + "epoch": 0.06, + "learning_rate": 3.0065359477124182e-06, + "logits/chosen": -2.8675389289855957, + "logits/rejected": -2.8082258701324463, + "logps/chosen": -291.63427734375, + "logps/rejected": -189.21340942382812, + "loss": 0.6859, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 0.03575144335627556, + "rewards/margins": 0.018049785867333412, + "rewards/rejected": 0.017701663076877594, + "step": 460 + }, + { + "epoch": 0.06, + "learning_rate": 3.071895424836602e-06, + "logits/chosen": -2.8911495208740234, + "logits/rejected": -2.9287312030792236, + "logps/chosen": -260.6916198730469, + "logps/rejected": -240.414794921875, + "loss": 0.6881, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.02840578183531761, + "rewards/margins": 0.010620644316077232, + "rewards/rejected": 0.01778513565659523, + "step": 470 + }, + { + "epoch": 0.06, + "learning_rate": 3.1372549019607846e-06, + "logits/chosen": -2.850501537322998, + "logits/rejected": -2.8640151023864746, + "logps/chosen": -284.7303161621094, + "logps/rejected": -307.1053161621094, + "loss": 0.6903, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.03151124715805054, + "rewards/margins": 0.004283557645976543, + "rewards/rejected": 0.02722769044339657, + "step": 480 + }, + { + "epoch": 0.06, + "learning_rate": 3.2026143790849674e-06, + "logits/chosen": -2.9750914573669434, + "logits/rejected": -2.8569235801696777, + "logps/chosen": -270.89971923828125, + "logps/rejected": -239.67733764648438, + "loss": 0.6867, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.031015124171972275, + "rewards/margins": 0.01439888495951891, + "rewards/rejected": 0.01661624014377594, + "step": 490 + }, + { + "epoch": 0.07, + "learning_rate": 3.2679738562091506e-06, + "logits/chosen": -2.913080930709839, + "logits/rejected": -2.9230728149414062, + "logps/chosen": -231.50753784179688, + "logps/rejected": -235.9259490966797, + "loss": 0.6861, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.03188391774892807, + "rewards/margins": 0.019386431202292442, + "rewards/rejected": 0.012497490271925926, + "step": 500 + }, + { + "epoch": 0.07, + "learning_rate": 3.3333333333333333e-06, + "logits/chosen": -2.949521541595459, + "logits/rejected": -2.937497615814209, + "logps/chosen": -352.7322082519531, + "logps/rejected": -253.5258331298828, + "loss": 0.6828, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 0.04447092488408089, + "rewards/margins": 0.024645883589982986, + "rewards/rejected": 0.0198250412940979, + "step": 510 + }, + { + "epoch": 0.07, + "learning_rate": 3.398692810457517e-06, + "logits/chosen": -2.9339003562927246, + "logits/rejected": -2.9172234535217285, + "logps/chosen": -302.03924560546875, + "logps/rejected": -246.05117797851562, + "loss": 0.6859, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.0391179658472538, + "rewards/margins": 0.015613268129527569, + "rewards/rejected": 0.023504700511693954, + "step": 520 + }, + { + "epoch": 0.07, + "learning_rate": 3.4640522875816997e-06, + "logits/chosen": -2.9032986164093018, + "logits/rejected": -2.9032576084136963, + "logps/chosen": -329.2999267578125, + "logps/rejected": -275.84149169921875, + "loss": 0.6858, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.04176343232393265, + "rewards/margins": 0.017148349434137344, + "rewards/rejected": 0.024615082889795303, + "step": 530 + }, + { + "epoch": 0.07, + "learning_rate": 3.529411764705883e-06, + "logits/chosen": -2.925100803375244, + "logits/rejected": -2.940062999725342, + "logps/chosen": -306.13970947265625, + "logps/rejected": -286.7759704589844, + "loss": 0.6865, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.04364129155874252, + "rewards/margins": 0.011506977491080761, + "rewards/rejected": 0.03213431313633919, + "step": 540 + }, + { + "epoch": 0.07, + "learning_rate": 3.5947712418300657e-06, + "logits/chosen": -2.7900989055633545, + "logits/rejected": -2.7149672508239746, + "logps/chosen": -275.6036682128906, + "logps/rejected": -221.5449981689453, + "loss": 0.6817, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.04563576728105545, + "rewards/margins": 0.026486817747354507, + "rewards/rejected": 0.019148947671055794, + "step": 550 + }, + { + "epoch": 0.07, + "learning_rate": 3.6601307189542484e-06, + "logits/chosen": -2.8575167655944824, + "logits/rejected": -2.919316053390503, + "logps/chosen": -268.87335205078125, + "logps/rejected": -253.87344360351562, + "loss": 0.6828, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.041931238025426865, + "rewards/margins": 0.02648743987083435, + "rewards/rejected": 0.015443804673850536, + "step": 560 + }, + { + "epoch": 0.07, + "learning_rate": 3.7254901960784316e-06, + "logits/chosen": -2.7716431617736816, + "logits/rejected": -2.86437726020813, + "logps/chosen": -313.5101013183594, + "logps/rejected": -239.9435577392578, + "loss": 0.6814, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 0.049281515181064606, + "rewards/margins": 0.032593242824077606, + "rewards/rejected": 0.016688276082277298, + "step": 570 + }, + { + "epoch": 0.08, + "learning_rate": 3.7908496732026144e-06, + "logits/chosen": -2.8355050086975098, + "logits/rejected": -2.7985503673553467, + "logps/chosen": -324.6849670410156, + "logps/rejected": -237.7136688232422, + "loss": 0.68, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04397376626729965, + "rewards/margins": 0.024682357907295227, + "rewards/rejected": 0.019291408360004425, + "step": 580 + }, + { + "epoch": 0.08, + "learning_rate": 3.856209150326798e-06, + "logits/chosen": -2.890514850616455, + "logits/rejected": -2.913870334625244, + "logps/chosen": -237.35205078125, + "logps/rejected": -230.11428833007812, + "loss": 0.6826, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.044465962797403336, + "rewards/margins": 0.021796438843011856, + "rewards/rejected": 0.02266952209174633, + "step": 590 + }, + { + "epoch": 0.08, + "learning_rate": 3.92156862745098e-06, + "logits/chosen": -2.872098207473755, + "logits/rejected": -2.8931877613067627, + "logps/chosen": -275.00848388671875, + "logps/rejected": -267.65802001953125, + "loss": 0.6812, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04082503542304039, + "rewards/margins": 0.026575183495879173, + "rewards/rejected": 0.014249855652451515, + "step": 600 + }, + { + "epoch": 0.08, + "learning_rate": 3.986928104575164e-06, + "logits/chosen": -2.755969285964966, + "logits/rejected": -2.7946555614471436, + "logps/chosen": -327.8434143066406, + "logps/rejected": -279.28509521484375, + "loss": 0.6817, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.04074610769748688, + "rewards/margins": 0.032059285789728165, + "rewards/rejected": 0.008686823770403862, + "step": 610 + }, + { + "epoch": 0.08, + "learning_rate": 4.052287581699347e-06, + "logits/chosen": -2.881762981414795, + "logits/rejected": -2.8726096153259277, + "logps/chosen": -274.4825744628906, + "logps/rejected": -254.6781768798828, + "loss": 0.6782, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.04438147693872452, + "rewards/margins": 0.020229998975992203, + "rewards/rejected": 0.024151481688022614, + "step": 620 + }, + { + "epoch": 0.08, + "learning_rate": 4.11764705882353e-06, + "logits/chosen": -2.784241199493408, + "logits/rejected": -2.8636996746063232, + "logps/chosen": -220.9033203125, + "logps/rejected": -205.02206420898438, + "loss": 0.681, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.029765665531158447, + "rewards/margins": 0.020760158076882362, + "rewards/rejected": 0.009005507454276085, + "step": 630 + }, + { + "epoch": 0.08, + "learning_rate": 4.183006535947713e-06, + "logits/chosen": -2.87015962600708, + "logits/rejected": -2.8887946605682373, + "logps/chosen": -290.60382080078125, + "logps/rejected": -308.30462646484375, + "loss": 0.6756, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 0.0495864637196064, + "rewards/margins": 0.036216553300619125, + "rewards/rejected": 0.013369908556342125, + "step": 640 + }, + { + "epoch": 0.09, + "learning_rate": 4.2483660130718954e-06, + "logits/chosen": -2.8120949268341064, + "logits/rejected": -2.882768154144287, + "logps/chosen": -310.6004333496094, + "logps/rejected": -279.83148193359375, + "loss": 0.6831, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.04262063279747963, + "rewards/margins": 0.016967860981822014, + "rewards/rejected": 0.025652769953012466, + "step": 650 + }, + { + "epoch": 0.09, + "learning_rate": 4.313725490196079e-06, + "logits/chosen": -2.878135919570923, + "logits/rejected": -2.896784543991089, + "logps/chosen": -235.76217651367188, + "logps/rejected": -198.4434814453125, + "loss": 0.6812, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 0.036157600581645966, + "rewards/margins": 0.02879006788134575, + "rewards/rejected": 0.007367536425590515, + "step": 660 + }, + { + "epoch": 0.09, + "learning_rate": 4.379084967320262e-06, + "logits/chosen": -2.8781301975250244, + "logits/rejected": -2.846781015396118, + "logps/chosen": -255.5959014892578, + "logps/rejected": -259.38726806640625, + "loss": 0.6795, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.04209792613983154, + "rewards/margins": 0.035201676189899445, + "rewards/rejected": 0.006896245293319225, + "step": 670 + }, + { + "epoch": 0.09, + "learning_rate": 4.444444444444444e-06, + "logits/chosen": -2.701145648956299, + "logits/rejected": -2.797853946685791, + "logps/chosen": -217.748046875, + "logps/rejected": -192.89178466796875, + "loss": 0.6766, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.03686373680830002, + "rewards/margins": 0.03340522199869156, + "rewards/rejected": 0.0034585162065923214, + "step": 680 + }, + { + "epoch": 0.09, + "learning_rate": 4.509803921568628e-06, + "logits/chosen": -2.799405097961426, + "logits/rejected": -2.800239324569702, + "logps/chosen": -224.0169219970703, + "logps/rejected": -208.5043487548828, + "loss": 0.6764, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 0.020510878413915634, + "rewards/margins": 0.03946324437856674, + "rewards/rejected": -0.01895236223936081, + "step": 690 + }, + { + "epoch": 0.09, + "learning_rate": 4.5751633986928105e-06, + "logits/chosen": -2.8377106189727783, + "logits/rejected": -2.7427613735198975, + "logps/chosen": -308.39202880859375, + "logps/rejected": -265.7181701660156, + "loss": 0.6783, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.01599673368036747, + "rewards/margins": 0.02599218487739563, + "rewards/rejected": -0.009995453990995884, + "step": 700 + }, + { + "epoch": 0.09, + "learning_rate": 4.640522875816994e-06, + "logits/chosen": -2.791973352432251, + "logits/rejected": -2.7882332801818848, + "logps/chosen": -321.95928955078125, + "logps/rejected": -268.7115783691406, + "loss": 0.6702, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.038942694664001465, + "rewards/margins": 0.06790758669376373, + "rewards/rejected": -0.02896488830447197, + "step": 710 + }, + { + "epoch": 0.09, + "learning_rate": 4.705882352941177e-06, + "logits/chosen": -2.8617916107177734, + "logits/rejected": -2.833081007003784, + "logps/chosen": -240.85415649414062, + "logps/rejected": -262.15325927734375, + "loss": 0.6694, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.03635946661233902, + "rewards/margins": 0.04408254846930504, + "rewards/rejected": -0.007723079528659582, + "step": 720 + }, + { + "epoch": 0.1, + "learning_rate": 4.77124183006536e-06, + "logits/chosen": -2.954502582550049, + "logits/rejected": -2.8966224193573, + "logps/chosen": -286.0347595214844, + "logps/rejected": -258.43487548828125, + "loss": 0.6733, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.03311768174171448, + "rewards/margins": 0.04485425353050232, + "rewards/rejected": -0.011736570857465267, + "step": 730 + }, + { + "epoch": 0.1, + "learning_rate": 4.836601307189543e-06, + "logits/chosen": -2.7532668113708496, + "logits/rejected": -2.789754867553711, + "logps/chosen": -287.6087341308594, + "logps/rejected": -268.8944396972656, + "loss": 0.6721, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.02871871553361416, + "rewards/margins": 0.06711134314537048, + "rewards/rejected": -0.03839262202382088, + "step": 740 + }, + { + "epoch": 0.1, + "learning_rate": 4.901960784313726e-06, + "logits/chosen": -2.722290277481079, + "logits/rejected": -2.757640838623047, + "logps/chosen": -299.148681640625, + "logps/rejected": -257.52972412109375, + "loss": 0.6725, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.006840469781309366, + "rewards/margins": 0.05222883075475693, + "rewards/rejected": -0.045388363301754, + "step": 750 + }, + { + "epoch": 0.1, + "learning_rate": 4.967320261437909e-06, + "logits/chosen": -2.7702317237854004, + "logits/rejected": -2.718785285949707, + "logps/chosen": -237.90707397460938, + "logps/rejected": -182.05593872070312, + "loss": 0.6783, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.016088716685771942, + "rewards/margins": 0.03843966871500015, + "rewards/rejected": -0.05452839285135269, + "step": 760 + }, + { + "epoch": 0.1, + "learning_rate": 4.99999347843947e-06, + "logits/chosen": -2.893463611602783, + "logits/rejected": -2.838412046432495, + "logps/chosen": -257.88311767578125, + "logps/rejected": -229.4723358154297, + "loss": 0.6723, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.025889281183481216, + "rewards/margins": 0.03446760028600693, + "rewards/rejected": -0.06035688519477844, + "step": 770 + }, + { + "epoch": 0.1, + "learning_rate": 4.999941306159375e-06, + "logits/chosen": -2.819159984588623, + "logits/rejected": -2.859947919845581, + "logps/chosen": -302.0553283691406, + "logps/rejected": -292.2879638671875, + "loss": 0.6758, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.051253754645586014, + "rewards/margins": 0.03177189081907272, + "rewards/rejected": -0.08302563428878784, + "step": 780 + }, + { + "epoch": 0.1, + "learning_rate": 4.999836962687967e-06, + "logits/chosen": -2.6668179035186768, + "logits/rejected": -2.667832851409912, + "logps/chosen": -262.34027099609375, + "logps/rejected": -295.4078674316406, + "loss": 0.6779, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.02748146653175354, + "rewards/margins": 0.035053275525569916, + "rewards/rejected": -0.06253473460674286, + "step": 790 + }, + { + "epoch": 0.1, + "learning_rate": 4.999680450202786e-06, + "logits/chosen": -2.7049105167388916, + "logits/rejected": -2.6771888732910156, + "logps/chosen": -341.10137939453125, + "logps/rejected": -301.953857421875, + "loss": 0.6637, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.009321318939328194, + "rewards/margins": 0.09051088988780975, + "rewards/rejected": -0.08118956536054611, + "step": 800 + }, + { + "epoch": 0.11, + "learning_rate": 4.999471771970087e-06, + "logits/chosen": -2.892444372177124, + "logits/rejected": -2.9383559226989746, + "logps/chosen": -229.4305877685547, + "logps/rejected": -203.98361206054688, + "loss": 0.6734, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0560198538005352, + "rewards/margins": 0.05636056512594223, + "rewards/rejected": -0.11238042265176773, + "step": 810 + }, + { + "epoch": 0.11, + "learning_rate": 4.999210932344767e-06, + "logits/chosen": -2.910004138946533, + "logits/rejected": -2.824584484100342, + "logps/chosen": -305.6726989746094, + "logps/rejected": -284.2978210449219, + "loss": 0.6624, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.042909882962703705, + "rewards/margins": 0.06963402032852173, + "rewards/rejected": -0.11254390329122543, + "step": 820 + }, + { + "epoch": 0.11, + "learning_rate": 4.998897936770281e-06, + "logits/chosen": -2.7205851078033447, + "logits/rejected": -2.6165709495544434, + "logps/chosen": -245.6926727294922, + "logps/rejected": -230.71853637695312, + "loss": 0.659, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07828576862812042, + "rewards/margins": 0.0336625911295414, + "rewards/rejected": -0.11194835603237152, + "step": 830 + }, + { + "epoch": 0.11, + "learning_rate": 4.998532791778521e-06, + "logits/chosen": -2.8109004497528076, + "logits/rejected": -2.7994141578674316, + "logps/chosen": -290.1483459472656, + "logps/rejected": -229.80569458007812, + "loss": 0.6652, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.052752308547496796, + "rewards/margins": 0.05021792650222778, + "rewards/rejected": -0.10297022759914398, + "step": 840 + }, + { + "epoch": 0.11, + "learning_rate": 4.9981155049896885e-06, + "logits/chosen": -2.822997570037842, + "logits/rejected": -2.8100199699401855, + "logps/chosen": -293.0641174316406, + "logps/rejected": -258.8555603027344, + "loss": 0.6549, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.032249774783849716, + "rewards/margins": 0.08926355838775635, + "rewards/rejected": -0.12151332944631577, + "step": 850 + }, + { + "epoch": 0.11, + "learning_rate": 4.997646085112126e-06, + "logits/chosen": -2.8823812007904053, + "logits/rejected": -2.889356851577759, + "logps/chosen": -285.6360778808594, + "logps/rejected": -243.662109375, + "loss": 0.669, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.032262224704027176, + "rewards/margins": 0.05970774218440056, + "rewards/rejected": -0.09196996688842773, + "step": 860 + }, + { + "epoch": 0.11, + "learning_rate": 4.997124541942141e-06, + "logits/chosen": -2.683950185775757, + "logits/rejected": -2.7463009357452393, + "logps/chosen": -272.642333984375, + "logps/rejected": -301.0301513671875, + "loss": 0.657, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.0004665583255700767, + "rewards/margins": 0.060696106404066086, + "rewards/rejected": -0.06116267293691635, + "step": 870 + }, + { + "epoch": 0.12, + "learning_rate": 4.996550886363801e-06, + "logits/chosen": -2.870196580886841, + "logits/rejected": -2.90971040725708, + "logps/chosen": -323.8336181640625, + "logps/rejected": -310.396728515625, + "loss": 0.6698, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.007984615862369537, + "rewards/margins": 0.04810230806469917, + "rewards/rejected": -0.040117692202329636, + "step": 880 + }, + { + "epoch": 0.12, + "learning_rate": 4.995925130348706e-06, + "logits/chosen": -2.7594635486602783, + "logits/rejected": -2.7156622409820557, + "logps/chosen": -339.4968566894531, + "logps/rejected": -281.2803649902344, + "loss": 0.6711, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.042974114418029785, + "rewards/margins": 0.05631721019744873, + "rewards/rejected": -0.09929133951663971, + "step": 890 + }, + { + "epoch": 0.12, + "learning_rate": 4.995247286955734e-06, + "logits/chosen": -2.8020763397216797, + "logits/rejected": -2.7794690132141113, + "logps/chosen": -290.42620849609375, + "logps/rejected": -286.94012451171875, + "loss": 0.6586, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.045134831219911575, + "rewards/margins": 0.07758808135986328, + "rewards/rejected": -0.12272290885448456, + "step": 900 + }, + { + "epoch": 0.12, + "learning_rate": 4.994517370330779e-06, + "logits/chosen": -2.7767722606658936, + "logits/rejected": -2.7736542224884033, + "logps/chosen": -271.8722229003906, + "logps/rejected": -282.7121276855469, + "loss": 0.6505, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.031097183004021645, + "rewards/margins": 0.10152481496334076, + "rewards/rejected": -0.13262200355529785, + "step": 910 + }, + { + "epoch": 0.12, + "learning_rate": 4.993735395706446e-06, + "logits/chosen": -2.7642810344696045, + "logits/rejected": -2.728177070617676, + "logps/chosen": -405.43310546875, + "logps/rejected": -429.303955078125, + "loss": 0.6646, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.11826081573963165, + "rewards/margins": 0.06041279435157776, + "rewards/rejected": -0.1786736249923706, + "step": 920 + }, + { + "epoch": 0.12, + "learning_rate": 4.992901379401737e-06, + "logits/chosen": -2.7699973583221436, + "logits/rejected": -2.6845641136169434, + "logps/chosen": -228.12570190429688, + "logps/rejected": -257.15802001953125, + "loss": 0.6511, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0507289357483387, + "rewards/margins": 0.11279155313968658, + "rewards/rejected": -0.16352050006389618, + "step": 930 + }, + { + "epoch": 0.12, + "learning_rate": 4.992015338821711e-06, + "logits/chosen": -2.811319351196289, + "logits/rejected": -2.7655045986175537, + "logps/chosen": -276.06842041015625, + "logps/rejected": -219.69436645507812, + "loss": 0.6588, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.087889164686203, + "rewards/margins": 0.07118016481399536, + "rewards/rejected": -0.15906934440135956, + "step": 940 + }, + { + "epoch": 0.12, + "learning_rate": 4.991077292457117e-06, + "logits/chosen": -2.8166513442993164, + "logits/rejected": -2.821939706802368, + "logps/chosen": -197.7022705078125, + "logps/rejected": -187.2440185546875, + "loss": 0.6406, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.017550457268953323, + "rewards/margins": 0.13717380166053772, + "rewards/rejected": -0.15472427010536194, + "step": 950 + }, + { + "epoch": 0.13, + "learning_rate": 4.990087259884016e-06, + "logits/chosen": -2.847860813140869, + "logits/rejected": -2.8105127811431885, + "logps/chosen": -293.509765625, + "logps/rejected": -290.2424621582031, + "loss": 0.665, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.05264557525515556, + "rewards/margins": 0.09494204819202423, + "rewards/rejected": -0.1475876271724701, + "step": 960 + }, + { + "epoch": 0.13, + "learning_rate": 4.989045261763362e-06, + "logits/chosen": -2.8128838539123535, + "logits/rejected": -2.7957558631896973, + "logps/chosen": -269.32257080078125, + "logps/rejected": -269.24151611328125, + "loss": 0.6493, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.04630335047841072, + "rewards/margins": 0.08781363815069199, + "rewards/rejected": -0.1341169774532318, + "step": 970 + }, + { + "epoch": 0.13, + "learning_rate": 4.98795131984058e-06, + "logits/chosen": -2.638960838317871, + "logits/rejected": -2.679980754852295, + "logps/chosen": -304.7935485839844, + "logps/rejected": -252.098388671875, + "loss": 0.6546, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.04581091180443764, + "rewards/margins": 0.11138496547937393, + "rewards/rejected": -0.15719588100910187, + "step": 980 + }, + { + "epoch": 0.13, + "learning_rate": 4.986805456945107e-06, + "logits/chosen": -2.7171177864074707, + "logits/rejected": -2.7990949153900146, + "logps/chosen": -306.76385498046875, + "logps/rejected": -291.9147033691406, + "loss": 0.6411, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.05044771358370781, + "rewards/margins": 0.18383342027664185, + "rewards/rejected": -0.23428115248680115, + "step": 990 + }, + { + "epoch": 0.13, + "learning_rate": 4.985607696989919e-06, + "logits/chosen": -2.778848648071289, + "logits/rejected": -2.735416889190674, + "logps/chosen": -302.1616516113281, + "logps/rejected": -293.0306091308594, + "loss": 0.668, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.013386363163590431, + "rewards/margins": 0.08964868634939194, + "rewards/rejected": -0.10303504765033722, + "step": 1000 + }, + { + "epoch": 0.13, + "learning_rate": 4.984358064971026e-06, + "logits/chosen": -2.837163209915161, + "logits/rejected": -2.8279595375061035, + "logps/chosen": -298.20404052734375, + "logps/rejected": -283.8133239746094, + "loss": 0.6451, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.039313506335020065, + "rewards/margins": 0.12387903034687042, + "rewards/rejected": -0.08456553518772125, + "step": 1010 + }, + { + "epoch": 0.13, + "learning_rate": 4.983056586966958e-06, + "logits/chosen": -2.7270495891571045, + "logits/rejected": -2.7269086837768555, + "logps/chosen": -296.46673583984375, + "logps/rejected": -290.07720947265625, + "loss": 0.6367, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 0.004742083139717579, + "rewards/margins": 0.09548323601484299, + "rewards/rejected": -0.09074117243289948, + "step": 1020 + }, + { + "epoch": 0.13, + "learning_rate": 4.981703290138215e-06, + "logits/chosen": -2.824183940887451, + "logits/rejected": -2.818276882171631, + "logps/chosen": -328.6833190917969, + "logps/rejected": -306.18365478515625, + "loss": 0.6431, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.08621495962142944, + "rewards/margins": 0.07550667226314545, + "rewards/rejected": -0.1617216318845749, + "step": 1030 + }, + { + "epoch": 0.14, + "learning_rate": 4.980298202726706e-06, + "logits/chosen": -2.8186137676239014, + "logits/rejected": -2.7979001998901367, + "logps/chosen": -311.40106201171875, + "logps/rejected": -313.767822265625, + "loss": 0.6632, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.06439457088708878, + "rewards/margins": 0.05475534871220589, + "rewards/rejected": -0.11914992332458496, + "step": 1040 + }, + { + "epoch": 0.14, + "learning_rate": 4.978841354055148e-06, + "logits/chosen": -2.7829668521881104, + "logits/rejected": -2.7322540283203125, + "logps/chosen": -236.6449432373047, + "logps/rejected": -241.97909545898438, + "loss": 0.6235, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.09969043731689453, + "rewards/margins": 0.14499905705451965, + "rewards/rejected": -0.24468949437141418, + "step": 1050 + }, + { + "epoch": 0.14, + "learning_rate": 4.977332774526471e-06, + "logits/chosen": -2.672950029373169, + "logits/rejected": -2.7831132411956787, + "logps/chosen": -245.9104461669922, + "logps/rejected": -272.1865234375, + "loss": 0.6499, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.062217772006988525, + "rewards/margins": 0.10472693294286728, + "rewards/rejected": -0.1669446974992752, + "step": 1060 + }, + { + "epoch": 0.14, + "learning_rate": 4.97577249562317e-06, + "logits/chosen": -2.8210768699645996, + "logits/rejected": -2.7688510417938232, + "logps/chosen": -260.833740234375, + "logps/rejected": -300.2538146972656, + "loss": 0.6339, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09711422771215439, + "rewards/margins": 0.12160750478506088, + "rewards/rejected": -0.21872171759605408, + "step": 1070 + }, + { + "epoch": 0.14, + "learning_rate": 4.974160549906652e-06, + "logits/chosen": -2.8435871601104736, + "logits/rejected": -2.8830039501190186, + "logps/chosen": -341.4996643066406, + "logps/rejected": -307.61663818359375, + "loss": 0.628, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.010590804740786552, + "rewards/margins": 0.13791541755199432, + "rewards/rejected": -0.12732459604740143, + "step": 1080 + }, + { + "epoch": 0.14, + "learning_rate": 4.972496971016559e-06, + "logits/chosen": -2.7579638957977295, + "logits/rejected": -2.735511064529419, + "logps/chosen": -264.26220703125, + "logps/rejected": -244.8957977294922, + "loss": 0.6322, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 0.029954945668578148, + "rewards/margins": 0.12742123007774353, + "rewards/rejected": -0.09746628999710083, + "step": 1090 + }, + { + "epoch": 0.14, + "learning_rate": 4.9707817936700635e-06, + "logits/chosen": -2.823887586593628, + "logits/rejected": -2.794739246368408, + "logps/chosen": -280.4059753417969, + "logps/rejected": -250.8085479736328, + "loss": 0.6697, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1311468780040741, + "rewards/margins": 0.04369507357478142, + "rewards/rejected": -0.174841970205307, + "step": 1100 + }, + { + "epoch": 0.15, + "learning_rate": 4.969015053661142e-06, + "logits/chosen": -2.7738354206085205, + "logits/rejected": -2.8087518215179443, + "logps/chosen": -307.9255065917969, + "logps/rejected": -286.568359375, + "loss": 0.6431, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.09180223196744919, + "rewards/margins": 0.14357972145080566, + "rewards/rejected": -0.23538196086883545, + "step": 1110 + }, + { + "epoch": 0.15, + "learning_rate": 4.967196787859835e-06, + "logits/chosen": -2.6887173652648926, + "logits/rejected": -2.7288146018981934, + "logps/chosen": -350.5284423828125, + "logps/rejected": -327.24774169921875, + "loss": 0.665, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.0915946513414383, + "rewards/margins": 0.013863353058695793, + "rewards/rejected": -0.10545799881219864, + "step": 1120 + }, + { + "epoch": 0.15, + "learning_rate": 4.965327034211469e-06, + "logits/chosen": -2.7602741718292236, + "logits/rejected": -2.723498821258545, + "logps/chosen": -272.59857177734375, + "logps/rejected": -213.2363739013672, + "loss": 0.6471, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.015425595454871655, + "rewards/margins": 0.12277853488922119, + "rewards/rejected": -0.13820412755012512, + "step": 1130 + }, + { + "epoch": 0.15, + "learning_rate": 4.96340583173587e-06, + "logits/chosen": -2.871131420135498, + "logits/rejected": -2.8382210731506348, + "logps/chosen": -331.5573425292969, + "logps/rejected": -303.3537292480469, + "loss": 0.6341, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.02129245176911354, + "rewards/margins": 0.15463818609714508, + "rewards/rejected": -0.13334573805332184, + "step": 1140 + }, + { + "epoch": 0.15, + "learning_rate": 4.96143322052655e-06, + "logits/chosen": -2.7604260444641113, + "logits/rejected": -2.766071319580078, + "logps/chosen": -253.5708465576172, + "logps/rejected": -287.50201416015625, + "loss": 0.6475, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.012628789059817791, + "rewards/margins": 0.10162287950515747, + "rewards/rejected": -0.11425165832042694, + "step": 1150 + }, + { + "epoch": 0.15, + "learning_rate": 4.959409241749864e-06, + "logits/chosen": -2.6584596633911133, + "logits/rejected": -2.6555018424987793, + "logps/chosen": -284.01031494140625, + "logps/rejected": -294.9400939941406, + "loss": 0.6496, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13796189427375793, + "rewards/margins": 0.13216018676757812, + "rewards/rejected": -0.27012208104133606, + "step": 1160 + }, + { + "epoch": 0.15, + "learning_rate": 4.957333937644159e-06, + "logits/chosen": -2.805464029312134, + "logits/rejected": -2.775489091873169, + "logps/chosen": -231.558837890625, + "logps/rejected": -246.1699981689453, + "loss": 0.626, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1663055121898651, + "rewards/margins": 0.1382933408021927, + "rewards/rejected": -0.304598867893219, + "step": 1170 + }, + { + "epoch": 0.15, + "learning_rate": 4.955207351518885e-06, + "logits/chosen": -2.8096632957458496, + "logits/rejected": -2.8571386337280273, + "logps/chosen": -310.173583984375, + "logps/rejected": -275.7320861816406, + "loss": 0.6637, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.04798074811697006, + "rewards/margins": 0.12220487743616104, + "rewards/rejected": -0.1701856255531311, + "step": 1180 + }, + { + "epoch": 0.16, + "learning_rate": 4.953029527753699e-06, + "logits/chosen": -2.6950483322143555, + "logits/rejected": -2.6458487510681152, + "logps/chosen": -303.374755859375, + "logps/rejected": -294.6081237792969, + "loss": 0.6576, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.0030715905595570803, + "rewards/margins": 0.1294359564781189, + "rewards/rejected": -0.1263643503189087, + "step": 1190 + }, + { + "epoch": 0.16, + "learning_rate": 4.95080051179753e-06, + "logits/chosen": -2.8290605545043945, + "logits/rejected": -2.814423084259033, + "logps/chosen": -240.0433807373047, + "logps/rejected": -255.37765502929688, + "loss": 0.6607, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 0.001875871792435646, + "rewards/margins": 0.14930638670921326, + "rewards/rejected": -0.14743052423000336, + "step": 1200 + }, + { + "epoch": 0.16, + "learning_rate": 4.948520350167637e-06, + "logits/chosen": -2.730070114135742, + "logits/rejected": -2.6461236476898193, + "logps/chosen": -343.88818359375, + "logps/rejected": -319.3534240722656, + "loss": 0.631, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.03022381290793419, + "rewards/margins": 0.14663995802402496, + "rewards/rejected": -0.11641611903905869, + "step": 1210 + }, + { + "epoch": 0.16, + "learning_rate": 4.946189090448639e-06, + "logits/chosen": -2.880946397781372, + "logits/rejected": -2.7842841148376465, + "logps/chosen": -335.5487060546875, + "logps/rejected": -312.669189453125, + "loss": 0.6076, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.03348718211054802, + "rewards/margins": 0.18006981909275055, + "rewards/rejected": -0.14658261835575104, + "step": 1220 + }, + { + "epoch": 0.16, + "learning_rate": 4.943806781291515e-06, + "logits/chosen": -2.687602996826172, + "logits/rejected": -2.674149751663208, + "logps/chosen": -300.13128662109375, + "logps/rejected": -349.22174072265625, + "loss": 0.6257, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06928616017103195, + "rewards/margins": 0.12933610379695892, + "rewards/rejected": -0.19862230122089386, + "step": 1230 + }, + { + "epoch": 0.16, + "learning_rate": 4.941373472412595e-06, + "logits/chosen": -2.758183479309082, + "logits/rejected": -2.7490592002868652, + "logps/chosen": -320.9246520996094, + "logps/rejected": -261.5167541503906, + "loss": 0.6353, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15161526203155518, + "rewards/margins": 0.18470586836338043, + "rewards/rejected": -0.336321085691452, + "step": 1240 + }, + { + "epoch": 0.16, + "learning_rate": 4.938889214592521e-06, + "logits/chosen": -2.6537938117980957, + "logits/rejected": -2.7111458778381348, + "logps/chosen": -225.9049072265625, + "logps/rejected": -289.18975830078125, + "loss": 0.6225, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21186769008636475, + "rewards/margins": 0.1365642547607422, + "rewards/rejected": -0.34843194484710693, + "step": 1250 + }, + { + "epoch": 0.16, + "learning_rate": 4.936354059675186e-06, + "logits/chosen": -2.8012726306915283, + "logits/rejected": -2.750591516494751, + "logps/chosen": -284.7136535644531, + "logps/rejected": -292.6429138183594, + "loss": 0.5986, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13322897255420685, + "rewards/margins": 0.18906542658805847, + "rewards/rejected": -0.3222944140434265, + "step": 1260 + }, + { + "epoch": 0.17, + "learning_rate": 4.933768060566654e-06, + "logits/chosen": -2.752645969390869, + "logits/rejected": -2.6327085494995117, + "logps/chosen": -294.1012268066406, + "logps/rejected": -268.6238708496094, + "loss": 0.6216, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11605393886566162, + "rewards/margins": 0.19025316834449768, + "rewards/rejected": -0.3063071370124817, + "step": 1270 + }, + { + "epoch": 0.17, + "learning_rate": 4.931131271234052e-06, + "logits/chosen": -2.746007204055786, + "logits/rejected": -2.756093978881836, + "logps/chosen": -302.9208679199219, + "logps/rejected": -326.42120361328125, + "loss": 0.6451, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07623710483312607, + "rewards/margins": 0.17326778173446655, + "rewards/rejected": -0.2495049238204956, + "step": 1280 + }, + { + "epoch": 0.17, + "learning_rate": 4.928443746704448e-06, + "logits/chosen": -2.76210355758667, + "logits/rejected": -2.666907548904419, + "logps/chosen": -274.781494140625, + "logps/rejected": -263.9569396972656, + "loss": 0.6144, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.009095259010791779, + "rewards/margins": 0.23592600226402283, + "rewards/rejected": -0.2450212687253952, + "step": 1290 + }, + { + "epoch": 0.17, + "learning_rate": 4.925705543063703e-06, + "logits/chosen": -2.745572090148926, + "logits/rejected": -2.7342967987060547, + "logps/chosen": -282.6916809082031, + "logps/rejected": -343.0442199707031, + "loss": 0.6367, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0001681298017501831, + "rewards/margins": 0.11077561229467392, + "rewards/rejected": -0.1109437495470047, + "step": 1300 + }, + { + "epoch": 0.17, + "learning_rate": 4.922916717455297e-06, + "logits/chosen": -2.647264003753662, + "logits/rejected": -2.682744264602661, + "logps/chosen": -249.7043914794922, + "logps/rejected": -263.6383056640625, + "loss": 0.6649, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.024844978004693985, + "rewards/margins": 0.07102416455745697, + "rewards/rejected": -0.04617919400334358, + "step": 1310 + }, + { + "epoch": 0.17, + "learning_rate": 4.920077328079136e-06, + "logits/chosen": -2.755326986312866, + "logits/rejected": -2.681166410446167, + "logps/chosen": -297.91131591796875, + "logps/rejected": -278.9104919433594, + "loss": 0.6113, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.014719530940055847, + "rewards/margins": 0.24292895197868347, + "rewards/rejected": -0.2282094657421112, + "step": 1320 + }, + { + "epoch": 0.17, + "learning_rate": 4.9171874341903445e-06, + "logits/chosen": -2.677175521850586, + "logits/rejected": -2.686843156814575, + "logps/chosen": -269.16278076171875, + "logps/rejected": -256.2588806152344, + "loss": 0.6117, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.18205681443214417, + "rewards/margins": 0.17718163132667542, + "rewards/rejected": -0.3592384457588196, + "step": 1330 + }, + { + "epoch": 0.18, + "learning_rate": 4.914247096098019e-06, + "logits/chosen": -2.647059679031372, + "logits/rejected": -2.6469664573669434, + "logps/chosen": -268.0980224609375, + "logps/rejected": -263.8585510253906, + "loss": 0.6107, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15819644927978516, + "rewards/margins": 0.19166433811187744, + "rewards/rejected": -0.3498607575893402, + "step": 1340 + }, + { + "epoch": 0.18, + "learning_rate": 4.911256375163977e-06, + "logits/chosen": -2.7415127754211426, + "logits/rejected": -2.729755163192749, + "logps/chosen": -325.60430908203125, + "logps/rejected": -279.28582763671875, + "loss": 0.6013, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.10827972739934921, + "rewards/margins": 0.21758142113685608, + "rewards/rejected": -0.3258611261844635, + "step": 1350 + }, + { + "epoch": 0.18, + "learning_rate": 4.908215333801474e-06, + "logits/chosen": -2.917020797729492, + "logits/rejected": -2.875368118286133, + "logps/chosen": -341.59417724609375, + "logps/rejected": -284.4775695800781, + "loss": 0.6644, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2299484759569168, + "rewards/margins": 0.12131299078464508, + "rewards/rejected": -0.3512614667415619, + "step": 1360 + }, + { + "epoch": 0.18, + "learning_rate": 4.9051240354739004e-06, + "logits/chosen": -2.7348055839538574, + "logits/rejected": -2.7037601470947266, + "logps/chosen": -320.5626525878906, + "logps/rejected": -323.69329833984375, + "loss": 0.6265, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.21234694123268127, + "rewards/margins": 0.1680133193731308, + "rewards/rejected": -0.3803602457046509, + "step": 1370 + }, + { + "epoch": 0.18, + "learning_rate": 4.901982544693457e-06, + "logits/chosen": -2.636758327484131, + "logits/rejected": -2.5937962532043457, + "logps/chosen": -183.79354858398438, + "logps/rejected": -258.00732421875, + "loss": 0.6105, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16744934022426605, + "rewards/margins": 0.1875566989183426, + "rewards/rejected": -0.35500603914260864, + "step": 1380 + }, + { + "epoch": 0.18, + "learning_rate": 4.898790927019809e-06, + "logits/chosen": -2.757171869277954, + "logits/rejected": -2.754363775253296, + "logps/chosen": -278.60546875, + "logps/rejected": -283.8389892578125, + "loss": 0.6109, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07067025452852249, + "rewards/margins": 0.10649889707565308, + "rewards/rejected": -0.17716915905475616, + "step": 1390 + }, + { + "epoch": 0.18, + "learning_rate": 4.895549249058718e-06, + "logits/chosen": -2.720224618911743, + "logits/rejected": -2.675530433654785, + "logps/chosen": -321.7203063964844, + "logps/rejected": -277.67974853515625, + "loss": 0.6092, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.016543136909604073, + "rewards/margins": 0.2174593210220337, + "rewards/rejected": -0.2340024709701538, + "step": 1400 + }, + { + "epoch": 0.18, + "learning_rate": 4.892257578460656e-06, + "logits/chosen": -2.677842617034912, + "logits/rejected": -2.664290189743042, + "logps/chosen": -218.2035675048828, + "logps/rejected": -224.66384887695312, + "loss": 0.6588, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.15112267434597015, + "rewards/margins": 0.13598957657814026, + "rewards/rejected": -0.2871122658252716, + "step": 1410 + }, + { + "epoch": 0.19, + "learning_rate": 4.888915983919383e-06, + "logits/chosen": -2.6758079528808594, + "logits/rejected": -2.7004618644714355, + "logps/chosen": -228.3094940185547, + "logps/rejected": -265.48486328125, + "loss": 0.5952, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08630862087011337, + "rewards/margins": 0.2088243067264557, + "rewards/rejected": -0.29513293504714966, + "step": 1420 + }, + { + "epoch": 0.19, + "learning_rate": 4.885524535170525e-06, + "logits/chosen": -2.6005783081054688, + "logits/rejected": -2.4832072257995605, + "logps/chosen": -231.92025756835938, + "logps/rejected": -235.0711212158203, + "loss": 0.6017, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.20524215698242188, + "rewards/margins": 0.1942833662033081, + "rewards/rejected": -0.39952555298805237, + "step": 1430 + }, + { + "epoch": 0.19, + "learning_rate": 4.882083302990113e-06, + "logits/chosen": -2.6818032264709473, + "logits/rejected": -2.6769537925720215, + "logps/chosen": -285.8664245605469, + "logps/rejected": -284.16156005859375, + "loss": 0.6581, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.28984886407852173, + "rewards/margins": 0.030770767480134964, + "rewards/rejected": -0.3206196427345276, + "step": 1440 + }, + { + "epoch": 0.19, + "learning_rate": 4.878592359193104e-06, + "logits/chosen": -2.692941665649414, + "logits/rejected": -2.6352081298828125, + "logps/chosen": -276.248046875, + "logps/rejected": -248.68612670898438, + "loss": 0.5998, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.29445046186447144, + "rewards/margins": 0.2336672991514206, + "rewards/rejected": -0.5281177759170532, + "step": 1450 + }, + { + "epoch": 0.19, + "learning_rate": 4.875051776631888e-06, + "logits/chosen": -2.64967679977417, + "logits/rejected": -2.7691729068756104, + "logps/chosen": -294.6110534667969, + "logps/rejected": -364.65570068359375, + "loss": 0.6243, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2757050096988678, + "rewards/margins": 0.1370132863521576, + "rewards/rejected": -0.4127182960510254, + "step": 1460 + }, + { + "epoch": 0.19, + "learning_rate": 4.871461629194764e-06, + "logits/chosen": -2.757596492767334, + "logits/rejected": -2.6786651611328125, + "logps/chosen": -336.01593017578125, + "logps/rejected": -315.3603515625, + "loss": 0.626, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08129008114337921, + "rewards/margins": 0.2590528726577759, + "rewards/rejected": -0.3403429388999939, + "step": 1470 + }, + { + "epoch": 0.19, + "learning_rate": 4.8678219918043984e-06, + "logits/chosen": -2.7759063243865967, + "logits/rejected": -2.7906689643859863, + "logps/chosen": -283.36651611328125, + "logps/rejected": -330.7609558105469, + "loss": 0.6293, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1509023904800415, + "rewards/margins": 0.17229530215263367, + "rewards/rejected": -0.3231976628303528, + "step": 1480 + }, + { + "epoch": 0.19, + "learning_rate": 4.864132940416262e-06, + "logits/chosen": -2.7591805458068848, + "logits/rejected": -2.7330336570739746, + "logps/chosen": -234.86569213867188, + "logps/rejected": -237.08261108398438, + "loss": 0.6472, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.22255483269691467, + "rewards/margins": 0.09442047029733658, + "rewards/rejected": -0.31697529554367065, + "step": 1490 + }, + { + "epoch": 0.2, + "learning_rate": 4.860394552017044e-06, + "logits/chosen": -2.828935146331787, + "logits/rejected": -2.750209331512451, + "logps/chosen": -312.0937194824219, + "logps/rejected": -307.0955505371094, + "loss": 0.5986, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.03175521641969681, + "rewards/margins": 0.20605985820293427, + "rewards/rejected": -0.23781506717205048, + "step": 1500 + }, + { + "epoch": 0.2, + "learning_rate": 4.856606904623047e-06, + "logits/chosen": -2.8377044200897217, + "logits/rejected": -2.7731568813323975, + "logps/chosen": -297.10894775390625, + "logps/rejected": -285.8016662597656, + "loss": 0.5955, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07155664265155792, + "rewards/margins": 0.2173517495393753, + "rewards/rejected": -0.2889083921909332, + "step": 1510 + }, + { + "epoch": 0.2, + "learning_rate": 4.852770077278557e-06, + "logits/chosen": -2.7048516273498535, + "logits/rejected": -2.6029093265533447, + "logps/chosen": -305.9083557128906, + "logps/rejected": -282.14971923828125, + "loss": 0.6173, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.17138591408729553, + "rewards/margins": 0.20368929207324982, + "rewards/rejected": -0.37507519125938416, + "step": 1520 + }, + { + "epoch": 0.2, + "learning_rate": 4.848884150054196e-06, + "logits/chosen": -2.572361946105957, + "logits/rejected": -2.5174009799957275, + "logps/chosen": -314.22760009765625, + "logps/rejected": -353.42315673828125, + "loss": 0.655, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.25344517827033997, + "rewards/margins": 0.11698782444000244, + "rewards/rejected": -0.3704330027103424, + "step": 1530 + }, + { + "epoch": 0.2, + "learning_rate": 4.8449492040452495e-06, + "logits/chosen": -2.7566611766815186, + "logits/rejected": -2.650944232940674, + "logps/chosen": -315.4299011230469, + "logps/rejected": -298.1441345214844, + "loss": 0.6212, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.25965338945388794, + "rewards/margins": 0.16350455582141876, + "rewards/rejected": -0.4231579303741455, + "step": 1540 + }, + { + "epoch": 0.2, + "learning_rate": 4.840965321369973e-06, + "logits/chosen": -2.717017650604248, + "logits/rejected": -2.8227248191833496, + "logps/chosen": -335.89678955078125, + "logps/rejected": -369.3492126464844, + "loss": 0.5908, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18783393502235413, + "rewards/margins": 0.28056249022483826, + "rewards/rejected": -0.4683963656425476, + "step": 1550 + }, + { + "epoch": 0.2, + "learning_rate": 4.8369325851678795e-06, + "logits/chosen": -2.669900417327881, + "logits/rejected": -2.695427417755127, + "logps/chosen": -303.61322021484375, + "logps/rejected": -267.2237243652344, + "loss": 0.584, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.08962138742208481, + "rewards/margins": 0.3239901661872864, + "rewards/rejected": -0.41361159086227417, + "step": 1560 + }, + { + "epoch": 0.21, + "learning_rate": 4.832851079598007e-06, + "logits/chosen": -2.6860480308532715, + "logits/rejected": -2.7314352989196777, + "logps/chosen": -292.6394348144531, + "logps/rejected": -296.482666015625, + "loss": 0.6553, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.23995356261730194, + "rewards/margins": 0.13955575227737427, + "rewards/rejected": -0.3795093297958374, + "step": 1570 + }, + { + "epoch": 0.21, + "learning_rate": 4.828720889837158e-06, + "logits/chosen": -2.8179385662078857, + "logits/rejected": -2.793445110321045, + "logps/chosen": -338.71771240234375, + "logps/rejected": -317.84600830078125, + "loss": 0.6828, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.1939702033996582, + "rewards/margins": 0.043380144983530045, + "rewards/rejected": -0.23735034465789795, + "step": 1580 + }, + { + "epoch": 0.21, + "learning_rate": 4.824542102078125e-06, + "logits/chosen": -2.684544801712036, + "logits/rejected": -2.69701886177063, + "logps/chosen": -304.6727600097656, + "logps/rejected": -278.4373474121094, + "loss": 0.5815, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.011012783274054527, + "rewards/margins": 0.2999088168144226, + "rewards/rejected": -0.3109216094017029, + "step": 1590 + }, + { + "epoch": 0.21, + "learning_rate": 4.820314803527888e-06, + "logits/chosen": -2.664644718170166, + "logits/rejected": -2.612112522125244, + "logps/chosen": -264.01153564453125, + "logps/rejected": -292.8648681640625, + "loss": 0.6086, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.22297194600105286, + "rewards/margins": 0.20747871696949005, + "rewards/rejected": -0.4304506182670593, + "step": 1600 + }, + { + "epoch": 0.21, + "learning_rate": 4.816039082405799e-06, + "logits/chosen": -2.5841476917266846, + "logits/rejected": -2.6593730449676514, + "logps/chosen": -277.6162414550781, + "logps/rejected": -287.6755676269531, + "loss": 0.6558, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.205702543258667, + "rewards/margins": 0.06671354919672012, + "rewards/rejected": -0.27241605520248413, + "step": 1610 + }, + { + "epoch": 0.21, + "learning_rate": 4.81171502794174e-06, + "logits/chosen": -2.6567516326904297, + "logits/rejected": -2.631348133087158, + "logps/chosen": -271.10296630859375, + "logps/rejected": -277.9122009277344, + "loss": 0.6361, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1482880860567093, + "rewards/margins": 0.19127169251441956, + "rewards/rejected": -0.33955979347229004, + "step": 1620 + }, + { + "epoch": 0.21, + "learning_rate": 4.8073427303742584e-06, + "logits/chosen": -2.6917014122009277, + "logits/rejected": -2.6560750007629395, + "logps/chosen": -287.2611389160156, + "logps/rejected": -231.60037231445312, + "loss": 0.5887, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09727681428194046, + "rewards/margins": 0.2694247364997864, + "rewards/rejected": -0.36670148372650146, + "step": 1630 + }, + { + "epoch": 0.21, + "learning_rate": 4.802922280948685e-06, + "logits/chosen": -2.655128240585327, + "logits/rejected": -2.6118204593658447, + "logps/chosen": -275.65118408203125, + "logps/rejected": -313.3224792480469, + "loss": 0.622, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09502645581960678, + "rewards/margins": 0.17091654241085052, + "rewards/rejected": -0.2659429907798767, + "step": 1640 + }, + { + "epoch": 0.22, + "learning_rate": 4.798453771915231e-06, + "logits/chosen": -2.7059059143066406, + "logits/rejected": -2.646460771560669, + "logps/chosen": -287.76092529296875, + "logps/rejected": -277.286865234375, + "loss": 0.5896, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.05740933492779732, + "rewards/margins": 0.2564863860607147, + "rewards/rejected": -0.31389567255973816, + "step": 1650 + }, + { + "epoch": 0.22, + "learning_rate": 4.793937296527062e-06, + "logits/chosen": -2.775752544403076, + "logits/rejected": -2.674415111541748, + "logps/chosen": -327.850830078125, + "logps/rejected": -313.1639099121094, + "loss": 0.6179, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.10228496789932251, + "rewards/margins": 0.2658602297306061, + "rewards/rejected": -0.3681451976299286, + "step": 1660 + }, + { + "epoch": 0.22, + "learning_rate": 4.78937294903835e-06, + "logits/chosen": -2.779148578643799, + "logits/rejected": -2.675994396209717, + "logps/chosen": -351.1578369140625, + "logps/rejected": -290.1517639160156, + "loss": 0.6293, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.09466046094894409, + "rewards/margins": 0.10398276150226593, + "rewards/rejected": -0.19864322245121002, + "step": 1670 + }, + { + "epoch": 0.22, + "learning_rate": 4.78476082470231e-06, + "logits/chosen": -2.767909526824951, + "logits/rejected": -2.756925106048584, + "logps/chosen": -302.81573486328125, + "logps/rejected": -312.35565185546875, + "loss": 0.6076, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.048796527087688446, + "rewards/margins": 0.21524441242218018, + "rewards/rejected": -0.2640409767627716, + "step": 1680 + }, + { + "epoch": 0.22, + "learning_rate": 4.780101019769212e-06, + "logits/chosen": -2.6890337467193604, + "logits/rejected": -2.6910290718078613, + "logps/chosen": -293.86968994140625, + "logps/rejected": -321.2923278808594, + "loss": 0.6275, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.06831858307123184, + "rewards/margins": 0.1262729912996292, + "rewards/rejected": -0.19459158182144165, + "step": 1690 + }, + { + "epoch": 0.22, + "learning_rate": 4.775393631484368e-06, + "logits/chosen": -2.777174472808838, + "logits/rejected": -2.7287497520446777, + "logps/chosen": -377.688720703125, + "logps/rejected": -403.62530517578125, + "loss": 0.5685, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06738211214542389, + "rewards/margins": 0.33521461486816406, + "rewards/rejected": -0.40259671211242676, + "step": 1700 + }, + { + "epoch": 0.22, + "learning_rate": 4.770638758086105e-06, + "logits/chosen": -2.7955563068389893, + "logits/rejected": -2.7565627098083496, + "logps/chosen": -322.578857421875, + "logps/rejected": -353.1636962890625, + "loss": 0.6419, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.16695697605609894, + "rewards/margins": 0.25171276926994324, + "rewards/rejected": -0.41866976022720337, + "step": 1710 + }, + { + "epoch": 0.23, + "learning_rate": 4.7658364988037184e-06, + "logits/chosen": -2.547750949859619, + "logits/rejected": -2.5859413146972656, + "logps/chosen": -260.44635009765625, + "logps/rejected": -299.1117858886719, + "loss": 0.5923, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.2033233344554901, + "rewards/margins": 0.2312351018190384, + "rewards/rejected": -0.4345583915710449, + "step": 1720 + }, + { + "epoch": 0.23, + "learning_rate": 4.760986953855395e-06, + "logits/chosen": -2.6293139457702637, + "logits/rejected": -2.5827693939208984, + "logps/chosen": -345.98992919921875, + "logps/rejected": -302.2580261230469, + "loss": 0.6219, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1974361389875412, + "rewards/margins": 0.1821739375591278, + "rewards/rejected": -0.3796100616455078, + "step": 1730 + }, + { + "epoch": 0.23, + "learning_rate": 4.756090224446127e-06, + "logits/chosen": -2.6410300731658936, + "logits/rejected": -2.5768771171569824, + "logps/chosen": -334.6339111328125, + "logps/rejected": -314.4638671875, + "loss": 0.6083, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17295120656490326, + "rewards/margins": 0.2973530888557434, + "rewards/rejected": -0.47030431032180786, + "step": 1740 + }, + { + "epoch": 0.23, + "learning_rate": 4.7511464127655945e-06, + "logits/chosen": -2.531867742538452, + "logits/rejected": -2.5611560344696045, + "logps/chosen": -261.2344970703125, + "logps/rejected": -305.4712219238281, + "loss": 0.6304, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.3530547320842743, + "rewards/margins": 0.15563586354255676, + "rewards/rejected": -0.508690595626831, + "step": 1750 + }, + { + "epoch": 0.23, + "learning_rate": 4.74615562198604e-06, + "logits/chosen": -2.623383045196533, + "logits/rejected": -2.608071804046631, + "logps/chosen": -271.5995788574219, + "logps/rejected": -261.22711181640625, + "loss": 0.6452, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.26331251859664917, + "rewards/margins": 0.21371789276599884, + "rewards/rejected": -0.4770304262638092, + "step": 1760 + }, + { + "epoch": 0.23, + "learning_rate": 4.741117956260107e-06, + "logits/chosen": -2.74521541595459, + "logits/rejected": -2.7192955017089844, + "logps/chosen": -363.3270263671875, + "logps/rejected": -349.2979431152344, + "loss": 0.6285, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2583506107330322, + "rewards/margins": 0.19173289835453033, + "rewards/rejected": -0.45008349418640137, + "step": 1770 + }, + { + "epoch": 0.23, + "learning_rate": 4.736033520718672e-06, + "logits/chosen": -2.708975315093994, + "logits/rejected": -2.62796950340271, + "logps/chosen": -256.0467224121094, + "logps/rejected": -234.60635375976562, + "loss": 0.613, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.29684996604919434, + "rewards/margins": 0.2133146971464157, + "rewards/rejected": -0.5101646780967712, + "step": 1780 + }, + { + "epoch": 0.23, + "learning_rate": 4.730902421468652e-06, + "logits/chosen": -2.72163724899292, + "logits/rejected": -2.676257610321045, + "logps/chosen": -292.6100769042969, + "logps/rejected": -270.1462707519531, + "loss": 0.5963, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.11978425830602646, + "rewards/margins": 0.30899566411972046, + "rewards/rejected": -0.4287799000740051, + "step": 1790 + }, + { + "epoch": 0.24, + "learning_rate": 4.7257247655907854e-06, + "logits/chosen": -2.6853718757629395, + "logits/rejected": -2.5619893074035645, + "logps/chosen": -307.2440185546875, + "logps/rejected": -281.82159423828125, + "loss": 0.6117, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14979851245880127, + "rewards/margins": 0.23921413719654083, + "rewards/rejected": -0.3890126645565033, + "step": 1800 + }, + { + "epoch": 0.24, + "learning_rate": 4.720500661137397e-06, + "logits/chosen": -2.618361711502075, + "logits/rejected": -2.6298348903656006, + "logps/chosen": -216.2972412109375, + "logps/rejected": -278.35882568359375, + "loss": 0.5912, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.19139596819877625, + "rewards/margins": 0.2812741696834564, + "rewards/rejected": -0.47267013788223267, + "step": 1810 + }, + { + "epoch": 0.24, + "learning_rate": 4.71523021713015e-06, + "logits/chosen": -2.756312608718872, + "logits/rejected": -2.6926093101501465, + "logps/chosen": -319.30950927734375, + "logps/rejected": -278.87603759765625, + "loss": 0.597, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.06740576773881912, + "rewards/margins": 0.1675378978252411, + "rewards/rejected": -0.2349436730146408, + "step": 1820 + }, + { + "epoch": 0.24, + "learning_rate": 4.709913543557761e-06, + "logits/chosen": -2.689087390899658, + "logits/rejected": -2.7317633628845215, + "logps/chosen": -337.2334899902344, + "logps/rejected": -316.35626220703125, + "loss": 0.6185, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1094675213098526, + "rewards/margins": 0.18793943524360657, + "rewards/rejected": -0.297406941652298, + "step": 1830 + }, + { + "epoch": 0.24, + "learning_rate": 4.704550751373715e-06, + "logits/chosen": -2.6714820861816406, + "logits/rejected": -2.6737353801727295, + "logps/chosen": -289.0609436035156, + "logps/rejected": -312.31292724609375, + "loss": 0.6554, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.12423040717840195, + "rewards/margins": 0.023334268480539322, + "rewards/rejected": -0.14756467938423157, + "step": 1840 + }, + { + "epoch": 0.24, + "learning_rate": 4.699141952493941e-06, + "logits/chosen": -2.6949803829193115, + "logits/rejected": -2.6727728843688965, + "logps/chosen": -294.4242248535156, + "logps/rejected": -266.7899169921875, + "loss": 0.5744, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06736332923173904, + "rewards/margins": 0.38571593165397644, + "rewards/rejected": -0.4530792832374573, + "step": 1850 + }, + { + "epoch": 0.24, + "learning_rate": 4.6936872597944814e-06, + "logits/chosen": -2.6007885932922363, + "logits/rejected": -2.542579174041748, + "logps/chosen": -281.5158996582031, + "logps/rejected": -312.28460693359375, + "loss": 0.6074, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1378696709871292, + "rewards/margins": 0.3318738341331482, + "rewards/rejected": -0.469743549823761, + "step": 1860 + }, + { + "epoch": 0.24, + "learning_rate": 4.688186787109136e-06, + "logits/chosen": -2.7046103477478027, + "logits/rejected": -2.680391311645508, + "logps/chosen": -259.2096252441406, + "logps/rejected": -260.27227783203125, + "loss": 0.6248, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1114644855260849, + "rewards/margins": 0.25269684195518494, + "rewards/rejected": -0.36416134238243103, + "step": 1870 + }, + { + "epoch": 0.25, + "learning_rate": 4.682640649227085e-06, + "logits/chosen": -2.5605578422546387, + "logits/rejected": -2.5646719932556152, + "logps/chosen": -298.17041015625, + "logps/rejected": -292.0390319824219, + "loss": 0.6108, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.11922509968280792, + "rewards/margins": 0.2266741245985031, + "rewards/rejected": -0.34589922428131104, + "step": 1880 + }, + { + "epoch": 0.25, + "learning_rate": 4.677048961890492e-06, + "logits/chosen": -2.531548023223877, + "logits/rejected": -2.5735363960266113, + "logps/chosen": -294.02911376953125, + "logps/rejected": -286.54901123046875, + "loss": 0.6073, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.08507753908634186, + "rewards/margins": 0.2579110264778137, + "rewards/rejected": -0.34298861026763916, + "step": 1890 + }, + { + "epoch": 0.25, + "learning_rate": 4.671411841792096e-06, + "logits/chosen": -2.741948366165161, + "logits/rejected": -2.741699695587158, + "logps/chosen": -361.36614990234375, + "logps/rejected": -290.7783203125, + "loss": 0.6137, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11754343658685684, + "rewards/margins": 0.2452048510313034, + "rewards/rejected": -0.36274832487106323, + "step": 1900 + }, + { + "epoch": 0.25, + "learning_rate": 4.665729406572764e-06, + "logits/chosen": -2.6238443851470947, + "logits/rejected": -2.5798392295837402, + "logps/chosen": -221.949951171875, + "logps/rejected": -245.45327758789062, + "loss": 0.6165, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.06824745237827301, + "rewards/margins": 0.2026282548904419, + "rewards/rejected": -0.2708757519721985, + "step": 1910 + }, + { + "epoch": 0.25, + "learning_rate": 4.660001774819048e-06, + "logits/chosen": -2.5808560848236084, + "logits/rejected": -2.5920627117156982, + "logps/chosen": -192.7483367919922, + "logps/rejected": -254.2974090576172, + "loss": 0.6229, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.11057275533676147, + "rewards/margins": 0.13108907639980316, + "rewards/rejected": -0.24166183173656464, + "step": 1920 + }, + { + "epoch": 0.25, + "learning_rate": 4.654229066060702e-06, + "logits/chosen": -2.71026349067688, + "logits/rejected": -2.7135469913482666, + "logps/chosen": -290.25140380859375, + "logps/rejected": -421.7354431152344, + "loss": 0.6206, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.18719585239887238, + "rewards/margins": 0.24483227729797363, + "rewards/rejected": -0.4320281147956848, + "step": 1930 + }, + { + "epoch": 0.25, + "learning_rate": 4.648411400768193e-06, + "logits/chosen": -2.6288414001464844, + "logits/rejected": -2.67307710647583, + "logps/chosen": -275.16961669921875, + "logps/rejected": -317.13226318359375, + "loss": 0.6248, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.09002985060214996, + "rewards/margins": 0.2158748209476471, + "rewards/rejected": -0.30590468645095825, + "step": 1940 + }, + { + "epoch": 0.26, + "learning_rate": 4.642548900350182e-06, + "logits/chosen": -2.5653369426727295, + "logits/rejected": -2.5764999389648438, + "logps/chosen": -307.288330078125, + "logps/rejected": -318.7103271484375, + "loss": 0.5834, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.07187150418758392, + "rewards/margins": 0.3741157650947571, + "rewards/rejected": -0.4459873139858246, + "step": 1950 + }, + { + "epoch": 0.26, + "learning_rate": 4.636641687150994e-06, + "logits/chosen": -2.672543525695801, + "logits/rejected": -2.6475882530212402, + "logps/chosen": -273.38629150390625, + "logps/rejected": -268.9444885253906, + "loss": 0.6402, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.11344961076974869, + "rewards/margins": 0.15282130241394043, + "rewards/rejected": -0.2662709057331085, + "step": 1960 + }, + { + "epoch": 0.26, + "learning_rate": 4.6306898844480615e-06, + "logits/chosen": -2.6119863986968994, + "logits/rejected": -2.5836076736450195, + "logps/chosen": -250.29446411132812, + "logps/rejected": -299.84393310546875, + "loss": 0.6036, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09432916343212128, + "rewards/margins": 0.20420034229755402, + "rewards/rejected": -0.2985295057296753, + "step": 1970 + }, + { + "epoch": 0.26, + "learning_rate": 4.624693616449358e-06, + "logits/chosen": -2.6079952716827393, + "logits/rejected": -2.5498125553131104, + "logps/chosen": -316.224365234375, + "logps/rejected": -280.0063171386719, + "loss": 0.6177, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.09396733343601227, + "rewards/margins": 0.21322092413902283, + "rewards/rejected": -0.3071882426738739, + "step": 1980 + }, + { + "epoch": 0.26, + "learning_rate": 4.6186530082908e-06, + "logits/chosen": -2.5833072662353516, + "logits/rejected": -2.627884864807129, + "logps/chosen": -283.31219482421875, + "logps/rejected": -310.50531005859375, + "loss": 0.6102, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.07193948328495026, + "rewards/margins": 0.12251697480678558, + "rewards/rejected": -0.19445645809173584, + "step": 1990 + }, + { + "epoch": 0.26, + "learning_rate": 4.612568186033633e-06, + "logits/chosen": -2.4248900413513184, + "logits/rejected": -2.510383129119873, + "logps/chosen": -252.7774658203125, + "logps/rejected": -208.8021240234375, + "loss": 0.6096, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.007582767400890589, + "rewards/margins": 0.2787870764732361, + "rewards/rejected": -0.28636986017227173, + "step": 2000 + }, + { + "epoch": 0.26, + "learning_rate": 4.6064392766618125e-06, + "logits/chosen": -2.569242238998413, + "logits/rejected": -2.593503952026367, + "logps/chosen": -278.3699645996094, + "logps/rejected": -289.50384521484375, + "loss": 0.6226, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2043885886669159, + "rewards/margins": 0.2912368178367615, + "rewards/rejected": -0.49562540650367737, + "step": 2010 + }, + { + "epoch": 0.26, + "learning_rate": 4.60026640807934e-06, + "logits/chosen": -2.65549373626709, + "logits/rejected": -2.6767234802246094, + "logps/chosen": -328.12628173828125, + "logps/rejected": -380.0748596191406, + "loss": 0.5954, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09818612784147263, + "rewards/margins": 0.28552350401878357, + "rewards/rejected": -0.3837096393108368, + "step": 2020 + }, + { + "epoch": 0.27, + "learning_rate": 4.594049709107604e-06, + "logits/chosen": -2.631639003753662, + "logits/rejected": -2.656987428665161, + "logps/chosen": -294.75909423828125, + "logps/rejected": -306.7351989746094, + "loss": 0.6244, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2435932159423828, + "rewards/margins": 0.16294609010219574, + "rewards/rejected": -0.40653929114341736, + "step": 2030 + }, + { + "epoch": 0.27, + "learning_rate": 4.587789309482687e-06, + "logits/chosen": -2.6387524604797363, + "logits/rejected": -2.6367218494415283, + "logps/chosen": -262.7574462890625, + "logps/rejected": -306.42279052734375, + "loss": 0.6419, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.4058169722557068, + "rewards/margins": 0.1947314441204071, + "rewards/rejected": -0.6005484461784363, + "step": 2040 + }, + { + "epoch": 0.27, + "learning_rate": 4.581485339852659e-06, + "logits/chosen": -2.6451351642608643, + "logits/rejected": -2.525254726409912, + "logps/chosen": -294.7330627441406, + "logps/rejected": -257.17169189453125, + "loss": 0.63, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.4744682312011719, + "rewards/margins": 0.11925798654556274, + "rewards/rejected": -0.5937261581420898, + "step": 2050 + }, + { + "epoch": 0.27, + "learning_rate": 4.5751379317748514e-06, + "logits/chosen": -2.61735463142395, + "logits/rejected": -2.6056129932403564, + "logps/chosen": -380.99456787109375, + "logps/rejected": -324.75494384765625, + "loss": 0.617, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3881939947605133, + "rewards/margins": 0.17417709529399872, + "rewards/rejected": -0.5623710751533508, + "step": 2060 + }, + { + "epoch": 0.27, + "learning_rate": 4.56874721771311e-06, + "logits/chosen": -2.672650098800659, + "logits/rejected": -2.6057205200195312, + "logps/chosen": -382.2373962402344, + "logps/rejected": -356.7444763183594, + "loss": 0.5966, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.29305747151374817, + "rewards/margins": 0.2830229103565216, + "rewards/rejected": -0.5760804414749146, + "step": 2070 + }, + { + "epoch": 0.27, + "learning_rate": 4.562313331035032e-06, + "logits/chosen": -2.511033058166504, + "logits/rejected": -2.5323522090911865, + "logps/chosen": -255.25381469726562, + "logps/rejected": -270.14080810546875, + "loss": 0.6579, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.31634312868118286, + "rewards/margins": 0.1493835151195526, + "rewards/rejected": -0.46572667360305786, + "step": 2080 + }, + { + "epoch": 0.27, + "learning_rate": 4.555836406009183e-06, + "logits/chosen": -2.562288761138916, + "logits/rejected": -2.595261573791504, + "logps/chosen": -307.12518310546875, + "logps/rejected": -275.62664794921875, + "loss": 0.5593, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.10846608877182007, + "rewards/margins": 0.2859782874584198, + "rewards/rejected": -0.39444440603256226, + "step": 2090 + }, + { + "epoch": 0.27, + "learning_rate": 4.5493165778022945e-06, + "logits/chosen": -2.5555379390716553, + "logits/rejected": -2.5553812980651855, + "logps/chosen": -262.2703857421875, + "logps/rejected": -292.42962646484375, + "loss": 0.6283, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.05403865501284599, + "rewards/margins": 0.2357197254896164, + "rewards/rejected": -0.2897583544254303, + "step": 2100 + }, + { + "epoch": 0.28, + "learning_rate": 4.542753982476443e-06, + "logits/chosen": -2.5372321605682373, + "logits/rejected": -2.5293571949005127, + "logps/chosen": -190.13748168945312, + "logps/rejected": -252.0640411376953, + "loss": 0.5592, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.03468228876590729, + "rewards/margins": 0.34676939249038696, + "rewards/rejected": -0.31208711862564087, + "step": 2110 + }, + { + "epoch": 0.28, + "learning_rate": 4.53614875698621e-06, + "logits/chosen": -2.6718530654907227, + "logits/rejected": -2.624863386154175, + "logps/chosen": -252.5389862060547, + "logps/rejected": -296.339111328125, + "loss": 0.6101, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.028590941801667213, + "rewards/margins": 0.25347763299942017, + "rewards/rejected": -0.28206855058670044, + "step": 2120 + }, + { + "epoch": 0.28, + "learning_rate": 4.529501039175824e-06, + "logits/chosen": -2.7555930614471436, + "logits/rejected": -2.6663403511047363, + "logps/chosen": -334.1688537597656, + "logps/rejected": -282.73583984375, + "loss": 0.6187, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.043031539767980576, + "rewards/margins": 0.25927868485450745, + "rewards/rejected": -0.3023102581501007, + "step": 2130 + }, + { + "epoch": 0.28, + "learning_rate": 4.522810967776287e-06, + "logits/chosen": -2.5664029121398926, + "logits/rejected": -2.607612133026123, + "logps/chosen": -274.611328125, + "logps/rejected": -273.4698791503906, + "loss": 0.6162, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16074858605861664, + "rewards/margins": 0.19277243316173553, + "rewards/rejected": -0.35352104902267456, + "step": 2140 + }, + { + "epoch": 0.28, + "learning_rate": 4.516078682402473e-06, + "logits/chosen": -2.594787359237671, + "logits/rejected": -2.6934938430786133, + "logps/chosen": -347.00531005859375, + "logps/rejected": -346.56829833984375, + "loss": 0.6192, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15159906446933746, + "rewards/margins": 0.2665919363498688, + "rewards/rejected": -0.41819101572036743, + "step": 2150 + }, + { + "epoch": 0.28, + "learning_rate": 4.509304323550221e-06, + "logits/chosen": -2.7246994972229004, + "logits/rejected": -2.6845176219940186, + "logps/chosen": -282.2721252441406, + "logps/rejected": -293.69183349609375, + "loss": 0.5828, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15215139091014862, + "rewards/margins": 0.19692817330360413, + "rewards/rejected": -0.34907954931259155, + "step": 2160 + }, + { + "epoch": 0.28, + "learning_rate": 4.502488032593398e-06, + "logits/chosen": -2.599320888519287, + "logits/rejected": -2.6243388652801514, + "logps/chosen": -285.3999938964844, + "logps/rejected": -285.7099609375, + "loss": 0.6159, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2641464173793793, + "rewards/margins": 0.25711730122566223, + "rewards/rejected": -0.5212637186050415, + "step": 2170 + }, + { + "epoch": 0.29, + "learning_rate": 4.495629951780951e-06, + "logits/chosen": -2.5960028171539307, + "logits/rejected": -2.637044906616211, + "logps/chosen": -265.2070007324219, + "logps/rejected": -312.5008239746094, + "loss": 0.6113, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.3445412516593933, + "rewards/margins": 0.24259276688098907, + "rewards/rejected": -0.587134063243866, + "step": 2180 + }, + { + "epoch": 0.29, + "learning_rate": 4.488730224233941e-06, + "logits/chosen": -2.498960018157959, + "logits/rejected": -2.470959186553955, + "logps/chosen": -261.1866455078125, + "logps/rejected": -286.18438720703125, + "loss": 0.6261, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.3730993866920471, + "rewards/margins": 0.17336183786392212, + "rewards/rejected": -0.5464612245559692, + "step": 2190 + }, + { + "epoch": 0.29, + "learning_rate": 4.481788993942547e-06, + "logits/chosen": -2.605865955352783, + "logits/rejected": -2.5414657592773438, + "logps/chosen": -273.25665283203125, + "logps/rejected": -297.3575134277344, + "loss": 0.5895, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.17746077477931976, + "rewards/margins": 0.26385658979415894, + "rewards/rejected": -0.4413173198699951, + "step": 2200 + }, + { + "epoch": 0.29, + "learning_rate": 4.474806405763076e-06, + "logits/chosen": -2.6408517360687256, + "logits/rejected": -2.654766082763672, + "logps/chosen": -335.73382568359375, + "logps/rejected": -394.8053894042969, + "loss": 0.6099, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18956801295280457, + "rewards/margins": 0.32702407240867615, + "rewards/rejected": -0.5165921449661255, + "step": 2210 + }, + { + "epoch": 0.29, + "learning_rate": 4.4677826054149235e-06, + "logits/chosen": -2.714592695236206, + "logits/rejected": -2.6655542850494385, + "logps/chosen": -307.36798095703125, + "logps/rejected": -319.69329833984375, + "loss": 0.6278, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.14881154894828796, + "rewards/margins": 0.2081369161605835, + "rewards/rejected": -0.35694849491119385, + "step": 2220 + }, + { + "epoch": 0.29, + "learning_rate": 4.460717739477543e-06, + "logits/chosen": -2.722327470779419, + "logits/rejected": -2.643770694732666, + "logps/chosen": -292.2183837890625, + "logps/rejected": -276.9352111816406, + "loss": 0.608, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.12699641287326813, + "rewards/margins": 0.22917278110980988, + "rewards/rejected": -0.3561691641807556, + "step": 2230 + }, + { + "epoch": 0.29, + "learning_rate": 4.4536119553873866e-06, + "logits/chosen": -2.6525509357452393, + "logits/rejected": -2.5999083518981934, + "logps/chosen": -264.67681884765625, + "logps/rejected": -323.0718994140625, + "loss": 0.6038, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13629117608070374, + "rewards/margins": 0.2666645348072052, + "rewards/rejected": -0.40295568108558655, + "step": 2240 + }, + { + "epoch": 0.29, + "learning_rate": 4.446465401434824e-06, + "logits/chosen": -2.734409809112549, + "logits/rejected": -2.800912380218506, + "logps/chosen": -304.42913818359375, + "logps/rejected": -317.63055419921875, + "loss": 0.6475, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.014048044569790363, + "rewards/margins": 0.23579058051109314, + "rewards/rejected": -0.22174255549907684, + "step": 2250 + }, + { + "epoch": 0.3, + "learning_rate": 4.43927822676105e-06, + "logits/chosen": -2.636152982711792, + "logits/rejected": -2.699537992477417, + "logps/chosen": -292.0948181152344, + "logps/rejected": -280.37127685546875, + "loss": 0.5936, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0466705784201622, + "rewards/margins": 0.27235180139541626, + "rewards/rejected": -0.31902235746383667, + "step": 2260 + }, + { + "epoch": 0.3, + "learning_rate": 4.432050581354972e-06, + "logits/chosen": -2.638198137283325, + "logits/rejected": -2.615269184112549, + "logps/chosen": -233.3173828125, + "logps/rejected": -236.16293334960938, + "loss": 0.5884, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.0751354768872261, + "rewards/margins": 0.2804257869720459, + "rewards/rejected": -0.3555612564086914, + "step": 2270 + }, + { + "epoch": 0.3, + "learning_rate": 4.424782616050078e-06, + "logits/chosen": -2.502224922180176, + "logits/rejected": -2.532297372817993, + "logps/chosen": -256.5329895019531, + "logps/rejected": -264.8097229003906, + "loss": 0.6088, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.0532631054520607, + "rewards/margins": 0.24914205074310303, + "rewards/rejected": -0.30240514874458313, + "step": 2280 + }, + { + "epoch": 0.3, + "learning_rate": 4.4174744825212954e-06, + "logits/chosen": -2.6726107597351074, + "logits/rejected": -2.677053451538086, + "logps/chosen": -314.2052307128906, + "logps/rejected": -336.2852783203125, + "loss": 0.5755, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.051678866147994995, + "rewards/margins": 0.33552059531211853, + "rewards/rejected": -0.3871994912624359, + "step": 2290 + }, + { + "epoch": 0.3, + "learning_rate": 4.410126333281815e-06, + "logits/chosen": -2.5050625801086426, + "logits/rejected": -2.568044900894165, + "logps/chosen": -312.2085876464844, + "logps/rejected": -248.065185546875, + "loss": 0.6049, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.148899644613266, + "rewards/margins": 0.2276383936405182, + "rewards/rejected": -0.3765380382537842, + "step": 2300 + }, + { + "epoch": 0.3, + "learning_rate": 4.402738321679918e-06, + "logits/chosen": -2.5214810371398926, + "logits/rejected": -2.5547168254852295, + "logps/chosen": -225.326904296875, + "logps/rejected": -262.37127685546875, + "loss": 0.6211, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.07122506946325302, + "rewards/margins": 0.2433677613735199, + "rewards/rejected": -0.3145928680896759, + "step": 2310 + }, + { + "epoch": 0.3, + "learning_rate": 4.395310601895772e-06, + "logits/chosen": -2.680088520050049, + "logits/rejected": -2.694715976715088, + "logps/chosen": -347.5968017578125, + "logps/rejected": -317.98797607421875, + "loss": 0.5905, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.058080125600099564, + "rewards/margins": 0.2747972011566162, + "rewards/rejected": -0.3328772783279419, + "step": 2320 + }, + { + "epoch": 0.3, + "learning_rate": 4.38784332893821e-06, + "logits/chosen": -2.61822509765625, + "logits/rejected": -2.62508225440979, + "logps/chosen": -309.4066467285156, + "logps/rejected": -281.98089599609375, + "loss": 0.6112, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.058225117623806, + "rewards/margins": 0.18333223462104797, + "rewards/rejected": -0.24155735969543457, + "step": 2330 + }, + { + "epoch": 0.31, + "learning_rate": 4.380336658641503e-06, + "logits/chosen": -2.738903284072876, + "logits/rejected": -2.714930772781372, + "logps/chosen": -298.86505126953125, + "logps/rejected": -351.3769836425781, + "loss": 0.5585, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.27263695001602173, + "rewards/margins": 0.23865346610546112, + "rewards/rejected": -0.5112903714179993, + "step": 2340 + }, + { + "epoch": 0.31, + "learning_rate": 4.372790747662101e-06, + "logits/chosen": -2.6900925636291504, + "logits/rejected": -2.6017818450927734, + "logps/chosen": -295.0585021972656, + "logps/rejected": -327.07720947265625, + "loss": 0.6036, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.25599241256713867, + "rewards/margins": 0.16895782947540283, + "rewards/rejected": -0.4249502122402191, + "step": 2350 + }, + { + "epoch": 0.31, + "learning_rate": 4.365205753475367e-06, + "logits/chosen": -2.600548505783081, + "logits/rejected": -2.595583200454712, + "logps/chosen": -321.8951110839844, + "logps/rejected": -297.0264587402344, + "loss": 0.5745, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.20295660197734833, + "rewards/margins": 0.32308974862098694, + "rewards/rejected": -0.5260463356971741, + "step": 2360 + }, + { + "epoch": 0.31, + "learning_rate": 4.35758183437229e-06, + "logits/chosen": -2.76790189743042, + "logits/rejected": -2.672685384750366, + "logps/chosen": -351.3072509765625, + "logps/rejected": -269.2875061035156, + "loss": 0.5585, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07739736884832382, + "rewards/margins": 0.2926849126815796, + "rewards/rejected": -0.3700822591781616, + "step": 2370 + }, + { + "epoch": 0.31, + "learning_rate": 4.3499191494561835e-06, + "logits/chosen": -2.5067715644836426, + "logits/rejected": -2.4817240238189697, + "logps/chosen": -318.60321044921875, + "logps/rejected": -330.294677734375, + "loss": 0.6151, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14703303575515747, + "rewards/margins": 0.286705881357193, + "rewards/rejected": -0.4337388873100281, + "step": 2380 + }, + { + "epoch": 0.31, + "learning_rate": 4.3422178586393615e-06, + "logits/chosen": -2.65042781829834, + "logits/rejected": -2.702307939529419, + "logps/chosen": -345.66900634765625, + "logps/rejected": -303.6436462402344, + "loss": 0.5587, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10787608474493027, + "rewards/margins": 0.35162174701690674, + "rewards/rejected": -0.4594978392124176, + "step": 2390 + }, + { + "epoch": 0.31, + "learning_rate": 4.334478122639804e-06, + "logits/chosen": -2.618354320526123, + "logits/rejected": -2.6288890838623047, + "logps/chosen": -348.8436584472656, + "logps/rejected": -257.6031799316406, + "loss": 0.6252, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.1821097582578659, + "rewards/margins": 0.15081281960010529, + "rewards/rejected": -0.3329225182533264, + "step": 2400 + }, + { + "epoch": 0.32, + "learning_rate": 4.3267001029778015e-06, + "logits/chosen": -2.5049855709075928, + "logits/rejected": -2.5256404876708984, + "logps/chosen": -303.19287109375, + "logps/rejected": -276.8233642578125, + "loss": 0.5768, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.05462324619293213, + "rewards/margins": 0.33293142914772034, + "rewards/rejected": -0.38755467534065247, + "step": 2410 + }, + { + "epoch": 0.32, + "learning_rate": 4.318883961972585e-06, + "logits/chosen": -2.7734265327453613, + "logits/rejected": -2.732790231704712, + "logps/chosen": -262.537109375, + "logps/rejected": -261.60992431640625, + "loss": 0.5923, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.18073582649230957, + "rewards/margins": 0.18429867923259735, + "rewards/rejected": -0.3650345206260681, + "step": 2420 + }, + { + "epoch": 0.32, + "learning_rate": 4.311029862738942e-06, + "logits/chosen": -2.5392842292785645, + "logits/rejected": -2.447587251663208, + "logps/chosen": -234.7491455078125, + "logps/rejected": -296.4173278808594, + "loss": 0.6278, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15868954360485077, + "rewards/margins": 0.24600186944007874, + "rewards/rejected": -0.4046913981437683, + "step": 2430 + }, + { + "epoch": 0.32, + "learning_rate": 4.303137969183804e-06, + "logits/chosen": -2.666558265686035, + "logits/rejected": -2.6016013622283936, + "logps/chosen": -304.29144287109375, + "logps/rejected": -345.33074951171875, + "loss": 0.5998, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15894465148448944, + "rewards/margins": 0.28240758180618286, + "rewards/rejected": -0.4413522779941559, + "step": 2440 + }, + { + "epoch": 0.32, + "learning_rate": 4.295208446002832e-06, + "logits/chosen": -2.5865111351013184, + "logits/rejected": -2.5046756267547607, + "logps/chosen": -230.9037628173828, + "logps/rejected": -231.08023071289062, + "loss": 0.5849, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.17778857052326202, + "rewards/margins": 0.2345859706401825, + "rewards/rejected": -0.4123745560646057, + "step": 2450 + }, + { + "epoch": 0.32, + "learning_rate": 4.287241458676981e-06, + "logits/chosen": -2.5890355110168457, + "logits/rejected": -2.5796329975128174, + "logps/chosen": -300.4346923828125, + "logps/rejected": -302.90716552734375, + "loss": 0.5272, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.08139663934707642, + "rewards/margins": 0.45793309807777405, + "rewards/rejected": -0.5393297076225281, + "step": 2460 + }, + { + "epoch": 0.32, + "learning_rate": 4.279237173469043e-06, + "logits/chosen": -2.575840473175049, + "logits/rejected": -2.5465903282165527, + "logps/chosen": -327.3525085449219, + "logps/rejected": -335.3431701660156, + "loss": 0.6049, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.21273550391197205, + "rewards/margins": 0.35986778140068054, + "rewards/rejected": -0.5726032853126526, + "step": 2470 + }, + { + "epoch": 0.32, + "learning_rate": 4.271195757420177e-06, + "logits/chosen": -2.612546443939209, + "logits/rejected": -2.5679924488067627, + "logps/chosen": -294.57562255859375, + "logps/rejected": -294.7464599609375, + "loss": 0.6151, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.3233681917190552, + "rewards/margins": 0.2548545002937317, + "rewards/rejected": -0.5782226920127869, + "step": 2480 + }, + { + "epoch": 0.33, + "learning_rate": 4.263117378346425e-06, + "logits/chosen": -2.7228329181671143, + "logits/rejected": -2.685037136077881, + "logps/chosen": -280.5471496582031, + "logps/rejected": -290.08734130859375, + "loss": 0.5829, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.17137329280376434, + "rewards/margins": 0.3288649618625641, + "rewards/rejected": -0.500238299369812, + "step": 2490 + }, + { + "epoch": 0.33, + "learning_rate": 4.255002204835208e-06, + "logits/chosen": -2.5566020011901855, + "logits/rejected": -2.4897701740264893, + "logps/chosen": -294.4997253417969, + "logps/rejected": -278.2536315917969, + "loss": 0.5891, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.26133105158805847, + "rewards/margins": 0.29285332560539246, + "rewards/rejected": -0.5541844367980957, + "step": 2500 + }, + { + "epoch": 0.33, + "learning_rate": 4.246850406241812e-06, + "logits/chosen": -2.6599020957946777, + "logits/rejected": -2.5850393772125244, + "logps/chosen": -371.8307800292969, + "logps/rejected": -401.79144287109375, + "loss": 0.5856, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.17885318398475647, + "rewards/margins": 0.34780776500701904, + "rewards/rejected": -0.5266609787940979, + "step": 2510 + }, + { + "epoch": 0.33, + "learning_rate": 4.2386621526858465e-06, + "logits/chosen": -2.6022260189056396, + "logits/rejected": -2.662214756011963, + "logps/chosen": -286.67987060546875, + "logps/rejected": -300.2646789550781, + "loss": 0.6027, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1267765760421753, + "rewards/margins": 0.29710814356803894, + "rewards/rejected": -0.423884779214859, + "step": 2520 + }, + { + "epoch": 0.33, + "learning_rate": 4.2304376150477015e-06, + "logits/chosen": -2.750474452972412, + "logits/rejected": -2.586371660232544, + "logps/chosen": -297.4253234863281, + "logps/rejected": -304.0201416015625, + "loss": 0.6329, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.15587542951107025, + "rewards/margins": 0.16625931859016418, + "rewards/rejected": -0.32213476300239563, + "step": 2530 + }, + { + "epoch": 0.33, + "learning_rate": 4.222176964964977e-06, + "logits/chosen": -2.681471109390259, + "logits/rejected": -2.567335367202759, + "logps/chosen": -292.96337890625, + "logps/rejected": -307.2474670410156, + "loss": 0.5617, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08317027986049652, + "rewards/margins": 0.4039889872074127, + "rewards/rejected": -0.4871593117713928, + "step": 2540 + }, + { + "epoch": 0.33, + "learning_rate": 4.213880374828903e-06, + "logits/chosen": -2.5859360694885254, + "logits/rejected": -2.645756244659424, + "logps/chosen": -301.61871337890625, + "logps/rejected": -295.96185302734375, + "loss": 0.575, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14113113284111023, + "rewards/margins": 0.3010835647583008, + "rewards/rejected": -0.4422147274017334, + "step": 2550 + }, + { + "epoch": 0.33, + "learning_rate": 4.2055480177807406e-06, + "logits/chosen": -2.6160552501678467, + "logits/rejected": -2.603322744369507, + "logps/chosen": -305.6348571777344, + "logps/rejected": -302.64752197265625, + "loss": 0.5837, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.11300075054168701, + "rewards/margins": 0.35259196162223816, + "rewards/rejected": -0.46559271216392517, + "step": 2560 + }, + { + "epoch": 0.34, + "learning_rate": 4.1971800677081696e-06, + "logits/chosen": -2.694335460662842, + "logits/rejected": -2.601956844329834, + "logps/chosen": -246.8837890625, + "logps/rejected": -275.73382568359375, + "loss": 0.5944, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.12262852489948273, + "rewards/margins": 0.37996551394462585, + "rewards/rejected": -0.5025940537452698, + "step": 2570 + }, + { + "epoch": 0.34, + "learning_rate": 4.188776699241661e-06, + "logits/chosen": -2.563056230545044, + "logits/rejected": -2.551748275756836, + "logps/chosen": -245.7527618408203, + "logps/rejected": -295.61627197265625, + "loss": 0.6244, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.27490201592445374, + "rewards/margins": 0.20333334803581238, + "rewards/rejected": -0.4782353341579437, + "step": 2580 + }, + { + "epoch": 0.34, + "learning_rate": 4.180338087750827e-06, + "logits/chosen": -2.5578365325927734, + "logits/rejected": -2.445997714996338, + "logps/chosen": -328.2390441894531, + "logps/rejected": -315.45623779296875, + "loss": 0.627, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3033788502216339, + "rewards/margins": 0.23796923458576202, + "rewards/rejected": -0.5413480997085571, + "step": 2590 + }, + { + "epoch": 0.34, + "learning_rate": 4.1718644093407704e-06, + "logits/chosen": -2.6223864555358887, + "logits/rejected": -2.6003425121307373, + "logps/chosen": -280.17181396484375, + "logps/rejected": -307.8580627441406, + "loss": 0.6391, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.15243183076381683, + "rewards/margins": 0.2037380486726761, + "rewards/rejected": -0.35616984963417053, + "step": 2600 + }, + { + "epoch": 0.34, + "learning_rate": 4.163355840848401e-06, + "logits/chosen": -2.6480376720428467, + "logits/rejected": -2.6332411766052246, + "logps/chosen": -287.12371826171875, + "logps/rejected": -306.5587158203125, + "loss": 0.598, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.10371170192956924, + "rewards/margins": 0.2772793769836426, + "rewards/rejected": -0.3809911012649536, + "step": 2610 + }, + { + "epoch": 0.34, + "learning_rate": 4.154812559838748e-06, + "logits/chosen": -2.655802011489868, + "logits/rejected": -2.6213440895080566, + "logps/chosen": -295.09576416015625, + "logps/rejected": -291.01861572265625, + "loss": 0.5772, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10176579654216766, + "rewards/margins": 0.3795744776725769, + "rewards/rejected": -0.48134031891822815, + "step": 2620 + }, + { + "epoch": 0.34, + "learning_rate": 4.146234744601259e-06, + "logits/chosen": -2.516303300857544, + "logits/rejected": -2.4881467819213867, + "logps/chosen": -328.66119384765625, + "logps/rejected": -295.0292663574219, + "loss": 0.5791, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.20398323237895966, + "rewards/margins": 0.32930490374565125, + "rewards/rejected": -0.5332881212234497, + "step": 2630 + }, + { + "epoch": 0.35, + "learning_rate": 4.137622574146071e-06, + "logits/chosen": -2.7313828468322754, + "logits/rejected": -2.6179592609405518, + "logps/chosen": -272.50152587890625, + "logps/rejected": -266.53436279296875, + "loss": 0.643, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.2861572206020355, + "rewards/margins": 0.20379790663719177, + "rewards/rejected": -0.4899550974369049, + "step": 2640 + }, + { + "epoch": 0.35, + "learning_rate": 4.12897622820028e-06, + "logits/chosen": -2.56074857711792, + "logits/rejected": -2.6304028034210205, + "logps/chosen": -348.1752624511719, + "logps/rejected": -356.3681335449219, + "loss": 0.6095, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2683704197406769, + "rewards/margins": 0.28429487347602844, + "rewards/rejected": -0.5526652932167053, + "step": 2650 + }, + { + "epoch": 0.35, + "learning_rate": 4.120295887204191e-06, + "logits/chosen": -2.4651379585266113, + "logits/rejected": -2.4907755851745605, + "logps/chosen": -326.99737548828125, + "logps/rejected": -277.8379821777344, + "loss": 0.6253, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.17758725583553314, + "rewards/margins": 0.28153958916664124, + "rewards/rejected": -0.4591267704963684, + "step": 2660 + }, + { + "epoch": 0.35, + "learning_rate": 4.111581732307548e-06, + "logits/chosen": -2.5480923652648926, + "logits/rejected": -2.5650529861450195, + "logps/chosen": -389.673583984375, + "logps/rejected": -335.89691162109375, + "loss": 0.6038, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2898654341697693, + "rewards/margins": 0.297260046005249, + "rewards/rejected": -0.5871254205703735, + "step": 2670 + }, + { + "epoch": 0.35, + "learning_rate": 4.1028339453657595e-06, + "logits/chosen": -2.5542445182800293, + "logits/rejected": -2.5450551509857178, + "logps/chosen": -300.4084777832031, + "logps/rejected": -293.3271789550781, + "loss": 0.5868, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1817891150712967, + "rewards/margins": 0.2439722716808319, + "rewards/rejected": -0.4257614016532898, + "step": 2680 + }, + { + "epoch": 0.35, + "learning_rate": 4.094052708936096e-06, + "logits/chosen": -2.6329503059387207, + "logits/rejected": -2.6375489234924316, + "logps/chosen": -302.6545104980469, + "logps/rejected": -266.0619201660156, + "loss": 0.5953, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.22319269180297852, + "rewards/margins": 0.2468569576740265, + "rewards/rejected": -0.4700496792793274, + "step": 2690 + }, + { + "epoch": 0.35, + "learning_rate": 4.0852382062738874e-06, + "logits/chosen": -2.5434086322784424, + "logits/rejected": -2.616213083267212, + "logps/chosen": -265.0691833496094, + "logps/rejected": -288.4176330566406, + "loss": 0.6006, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.295448362827301, + "rewards/margins": 0.2674034833908081, + "rewards/rejected": -0.5628517866134644, + "step": 2700 + }, + { + "epoch": 0.35, + "learning_rate": 4.076390621328693e-06, + "logits/chosen": -2.605640411376953, + "logits/rejected": -2.5978264808654785, + "logps/chosen": -278.6802062988281, + "logps/rejected": -245.44406127929688, + "loss": 0.6441, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.22599074244499207, + "rewards/margins": 0.18402789533138275, + "rewards/rejected": -0.4100186228752136, + "step": 2710 + }, + { + "epoch": 0.36, + "learning_rate": 4.067510138740467e-06, + "logits/chosen": -2.5916764736175537, + "logits/rejected": -2.516136407852173, + "logps/chosen": -295.1086120605469, + "logps/rejected": -310.83648681640625, + "loss": 0.6608, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.31419694423675537, + "rewards/margins": 0.19866469502449036, + "rewards/rejected": -0.5128616094589233, + "step": 2720 + }, + { + "epoch": 0.36, + "learning_rate": 4.058596943835703e-06, + "logits/chosen": -2.6483025550842285, + "logits/rejected": -2.579637289047241, + "logps/chosen": -307.8289794921875, + "logps/rejected": -359.41021728515625, + "loss": 0.5861, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.23769135773181915, + "rewards/margins": 0.35159462690353394, + "rewards/rejected": -0.5892860293388367, + "step": 2730 + }, + { + "epoch": 0.36, + "learning_rate": 4.049651222623568e-06, + "logits/chosen": -2.4489874839782715, + "logits/rejected": -2.464409112930298, + "logps/chosen": -321.61859130859375, + "logps/rejected": -280.51934814453125, + "loss": 0.6288, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.17855051159858704, + "rewards/margins": 0.26512032747268677, + "rewards/rejected": -0.4436708390712738, + "step": 2740 + }, + { + "epoch": 0.36, + "learning_rate": 4.040673161792014e-06, + "logits/chosen": -2.571610927581787, + "logits/rejected": -2.623656749725342, + "logps/chosen": -244.89987182617188, + "logps/rejected": -275.18072509765625, + "loss": 0.5884, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1748734712600708, + "rewards/margins": 0.3799809217453003, + "rewards/rejected": -0.5548543930053711, + "step": 2750 + }, + { + "epoch": 0.36, + "learning_rate": 4.031662948703896e-06, + "logits/chosen": -2.6747171878814697, + "logits/rejected": -2.6468615531921387, + "logps/chosen": -337.708251953125, + "logps/rejected": -333.896240234375, + "loss": 0.5863, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.120061494410038, + "rewards/margins": 0.35260099172592163, + "rewards/rejected": -0.4726625084877014, + "step": 2760 + }, + { + "epoch": 0.36, + "learning_rate": 4.022620771393047e-06, + "logits/chosen": -2.544447660446167, + "logits/rejected": -2.5561606884002686, + "logps/chosen": -316.19488525390625, + "logps/rejected": -294.08282470703125, + "loss": 0.5838, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.252451092004776, + "rewards/margins": 0.18410982191562653, + "rewards/rejected": -0.43656086921691895, + "step": 2770 + }, + { + "epoch": 0.36, + "learning_rate": 4.013546818560362e-06, + "logits/chosen": -2.668536901473999, + "logits/rejected": -2.6385653018951416, + "logps/chosen": -361.6271667480469, + "logps/rejected": -322.5511169433594, + "loss": 0.5876, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.25951510667800903, + "rewards/margins": 0.21376793086528778, + "rewards/rejected": -0.473283052444458, + "step": 2780 + }, + { + "epoch": 0.37, + "learning_rate": 4.00444127956986e-06, + "logits/chosen": -2.5495948791503906, + "logits/rejected": -2.532189130783081, + "logps/chosen": -326.3888244628906, + "logps/rejected": -347.3705139160156, + "loss": 0.5931, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.21870854496955872, + "rewards/margins": 0.30374085903167725, + "rewards/rejected": -0.5224493741989136, + "step": 2790 + }, + { + "epoch": 0.37, + "learning_rate": 3.9953043444447255e-06, + "logits/chosen": -2.64678955078125, + "logits/rejected": -2.6592841148376465, + "logps/chosen": -333.729736328125, + "logps/rejected": -371.07586669921875, + "loss": 0.6334, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.24805398285388947, + "rewards/margins": 0.23774032294750214, + "rewards/rejected": -0.4857942461967468, + "step": 2800 + }, + { + "epoch": 0.37, + "learning_rate": 3.986136203863355e-06, + "logits/chosen": -2.6806514263153076, + "logits/rejected": -2.665834903717041, + "logps/chosen": -304.5228576660156, + "logps/rejected": -318.13751220703125, + "loss": 0.5897, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.2563796043395996, + "rewards/margins": 0.24397775530815125, + "rewards/rejected": -0.5003573298454285, + "step": 2810 + }, + { + "epoch": 0.37, + "learning_rate": 3.976937049155365e-06, + "logits/chosen": -2.6606526374816895, + "logits/rejected": -2.564753293991089, + "logps/chosen": -314.8397521972656, + "logps/rejected": -387.33428955078125, + "loss": 0.6089, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16455021500587463, + "rewards/margins": 0.2586231231689453, + "rewards/rejected": -0.42317327857017517, + "step": 2820 + }, + { + "epoch": 0.37, + "learning_rate": 3.967707072297608e-06, + "logits/chosen": -2.628833532333374, + "logits/rejected": -2.5874598026275635, + "logps/chosen": -237.16250610351562, + "logps/rejected": -254.37435913085938, + "loss": 0.5682, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.02314874157309532, + "rewards/margins": 0.287275105714798, + "rewards/rejected": -0.3104238510131836, + "step": 2830 + }, + { + "epoch": 0.37, + "learning_rate": 3.958446465910159e-06, + "logits/chosen": -2.5451273918151855, + "logits/rejected": -2.622796058654785, + "logps/chosen": -285.39422607421875, + "logps/rejected": -320.67919921875, + "loss": 0.6052, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12590385973453522, + "rewards/margins": 0.18081295490264893, + "rewards/rejected": -0.30671682953834534, + "step": 2840 + }, + { + "epoch": 0.37, + "learning_rate": 3.9491554232523066e-06, + "logits/chosen": -2.461094856262207, + "logits/rejected": -2.485332489013672, + "logps/chosen": -262.26190185546875, + "logps/rejected": -277.8798828125, + "loss": 0.619, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09473375976085663, + "rewards/margins": 0.26800814270973206, + "rewards/rejected": -0.3627418875694275, + "step": 2850 + }, + { + "epoch": 0.37, + "learning_rate": 3.939834138218505e-06, + "logits/chosen": -2.543159008026123, + "logits/rejected": -2.5069081783294678, + "logps/chosen": -239.6870574951172, + "logps/rejected": -274.2434387207031, + "loss": 0.5939, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.07561665028333664, + "rewards/margins": 0.23530197143554688, + "rewards/rejected": -0.3109186291694641, + "step": 2860 + }, + { + "epoch": 0.38, + "learning_rate": 3.930482805334339e-06, + "logits/chosen": -2.613980293273926, + "logits/rejected": -2.5922513008117676, + "logps/chosen": -265.4554748535156, + "logps/rejected": -252.3996124267578, + "loss": 0.6082, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.189830020070076, + "rewards/margins": 0.1698548048734665, + "rewards/rejected": -0.35968488454818726, + "step": 2870 + }, + { + "epoch": 0.38, + "learning_rate": 3.921101619752464e-06, + "logits/chosen": -2.606279134750366, + "logits/rejected": -2.6501641273498535, + "logps/chosen": -312.08782958984375, + "logps/rejected": -354.805419921875, + "loss": 0.5963, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15292449295520782, + "rewards/margins": 0.2811005413532257, + "rewards/rejected": -0.43402498960494995, + "step": 2880 + }, + { + "epoch": 0.38, + "learning_rate": 3.911690777248525e-06, + "logits/chosen": -2.5766007900238037, + "logits/rejected": -2.56482195854187, + "logps/chosen": -257.36468505859375, + "logps/rejected": -266.5788879394531, + "loss": 0.6007, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1831916868686676, + "rewards/margins": 0.14657798409461975, + "rewards/rejected": -0.32976967096328735, + "step": 2890 + }, + { + "epoch": 0.38, + "learning_rate": 3.902250474217079e-06, + "logits/chosen": -2.509827136993408, + "logits/rejected": -2.5269761085510254, + "logps/chosen": -199.52395629882812, + "logps/rejected": -279.4906921386719, + "loss": 0.5741, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.13312488794326782, + "rewards/margins": 0.4061674177646637, + "rewards/rejected": -0.5392922759056091, + "step": 2900 + }, + { + "epoch": 0.38, + "learning_rate": 3.892780907667495e-06, + "logits/chosen": -2.6970276832580566, + "logits/rejected": -2.577418565750122, + "logps/chosen": -323.4083557128906, + "logps/rejected": -338.27716064453125, + "loss": 0.5847, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.06340484321117401, + "rewards/margins": 0.26021626591682434, + "rewards/rejected": -0.32362112402915955, + "step": 2910 + }, + { + "epoch": 0.38, + "learning_rate": 3.883282275219837e-06, + "logits/chosen": -2.563586711883545, + "logits/rejected": -2.5951290130615234, + "logps/chosen": -277.5760192871094, + "logps/rejected": -300.0455017089844, + "loss": 0.5965, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.168487086892128, + "rewards/margins": 0.26566335558891296, + "rewards/rejected": -0.4341503977775574, + "step": 2920 + }, + { + "epoch": 0.38, + "learning_rate": 3.873754775100751e-06, + "logits/chosen": -2.4823224544525146, + "logits/rejected": -2.4202988147735596, + "logps/chosen": -268.9900207519531, + "logps/rejected": -318.63873291015625, + "loss": 0.5996, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.21859273314476013, + "rewards/margins": 0.2954866886138916, + "rewards/rejected": -0.5140794515609741, + "step": 2930 + }, + { + "epoch": 0.38, + "learning_rate": 3.8641986061393145e-06, + "logits/chosen": -2.676154613494873, + "logits/rejected": -2.6082122325897217, + "logps/chosen": -307.00872802734375, + "logps/rejected": -281.6736755371094, + "loss": 0.5508, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08748184889554977, + "rewards/margins": 0.4146133363246918, + "rewards/rejected": -0.5020952224731445, + "step": 2940 + }, + { + "epoch": 0.39, + "learning_rate": 3.854613967762898e-06, + "logits/chosen": -2.4909651279449463, + "logits/rejected": -2.5078840255737305, + "logps/chosen": -279.0023498535156, + "logps/rejected": -332.1703186035156, + "loss": 0.5596, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.09293212741613388, + "rewards/margins": 0.3573867678642273, + "rewards/rejected": -0.4503189027309418, + "step": 2950 + }, + { + "epoch": 0.39, + "learning_rate": 3.845001059992999e-06, + "logits/chosen": -2.661405086517334, + "logits/rejected": -2.599020004272461, + "logps/chosen": -331.3071594238281, + "logps/rejected": -333.71746826171875, + "loss": 0.6092, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.19790147244930267, + "rewards/margins": 0.24655024707317352, + "rewards/rejected": -0.4444517195224762, + "step": 2960 + }, + { + "epoch": 0.39, + "learning_rate": 3.835360083441067e-06, + "logits/chosen": -2.714463472366333, + "logits/rejected": -2.6763317584991455, + "logps/chosen": -273.2596740722656, + "logps/rejected": -295.5934143066406, + "loss": 0.6252, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3185594081878662, + "rewards/margins": 0.18595674633979797, + "rewards/rejected": -0.5045161247253418, + "step": 2970 + }, + { + "epoch": 0.39, + "learning_rate": 3.825691239304318e-06, + "logits/chosen": -2.5877997875213623, + "logits/rejected": -2.5963594913482666, + "logps/chosen": -289.68316650390625, + "logps/rejected": -308.5525817871094, + "loss": 0.6279, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.20810465514659882, + "rewards/margins": 0.3502613306045532, + "rewards/rejected": -0.5583660006523132, + "step": 2980 + }, + { + "epoch": 0.39, + "learning_rate": 3.8159947293615385e-06, + "logits/chosen": -2.6402833461761475, + "logits/rejected": -2.645634412765503, + "logps/chosen": -328.13360595703125, + "logps/rejected": -304.3066101074219, + "loss": 0.6314, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.23268917202949524, + "rewards/margins": 0.17129898071289062, + "rewards/rejected": -0.40398818254470825, + "step": 2990 + }, + { + "epoch": 0.39, + "learning_rate": 3.806270755968866e-06, + "logits/chosen": -2.6010279655456543, + "logits/rejected": -2.635829448699951, + "logps/chosen": -193.94699096679688, + "logps/rejected": -273.6504211425781, + "loss": 0.5526, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.020387057214975357, + "rewards/margins": 0.34069180488586426, + "rewards/rejected": -0.3610789179801941, + "step": 3000 + }, + { + "epoch": 0.39, + "eval_logits/chosen": -2.592276096343994, + "eval_logits/rejected": -2.565138101577759, + "eval_logps/chosen": -299.8006591796875, + "eval_logps/rejected": -291.83782958984375, + "eval_loss": 0.6012948751449585, + "eval_rewards/accuracies": 0.7039999961853027, + "eval_rewards/chosen": -0.15081314742565155, + "eval_rewards/margins": 0.2546119689941406, + "eval_rewards/rejected": -0.40542513132095337, + "eval_runtime": 469.9385, + "eval_samples_per_second": 4.256, + "eval_steps_per_second": 1.064, + "step": 3000 + }, + { + "epoch": 0.39, + "learning_rate": 3.7965195220555784e-06, + "logits/chosen": -2.6464450359344482, + "logits/rejected": -2.64005446434021, + "logps/chosen": -304.0558166503906, + "logps/rejected": -334.44268798828125, + "loss": 0.5749, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.06986355036497116, + "rewards/margins": 0.3272497355937958, + "rewards/rejected": -0.39711323380470276, + "step": 3010 + }, + { + "epoch": 0.4, + "learning_rate": 3.786741231119847e-06, + "logits/chosen": -2.5847103595733643, + "logits/rejected": -2.610323667526245, + "logps/chosen": -271.672607421875, + "logps/rejected": -317.8426208496094, + "loss": 0.5225, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.07842519134283066, + "rewards/margins": 0.4202388823032379, + "rewards/rejected": -0.4986640512943268, + "step": 3020 + }, + { + "epoch": 0.4, + "learning_rate": 3.7769360872244992e-06, + "logits/chosen": -2.639026641845703, + "logits/rejected": -2.666348934173584, + "logps/chosen": -333.561767578125, + "logps/rejected": -347.01654052734375, + "loss": 0.6074, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.2456272393465042, + "rewards/margins": 0.20590810477733612, + "rewards/rejected": -0.4515353739261627, + "step": 3030 + }, + { + "epoch": 0.4, + "learning_rate": 3.767104294992754e-06, + "logits/chosen": -2.6955556869506836, + "logits/rejected": -2.588688373565674, + "logps/chosen": -276.519775390625, + "logps/rejected": -271.98175048828125, + "loss": 0.6202, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1983802318572998, + "rewards/margins": 0.18987175822257996, + "rewards/rejected": -0.38825201988220215, + "step": 3040 + }, + { + "epoch": 0.4, + "learning_rate": 3.7572460596039524e-06, + "logits/chosen": -2.574049949645996, + "logits/rejected": -2.558225631713867, + "logps/chosen": -259.74078369140625, + "logps/rejected": -285.2066955566406, + "loss": 0.639, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2721092998981476, + "rewards/margins": 0.23243455588817596, + "rewards/rejected": -0.5045438408851624, + "step": 3050 + }, + { + "epoch": 0.4, + "learning_rate": 3.74736158678928e-06, + "logits/chosen": -2.3812010288238525, + "logits/rejected": -2.39877986907959, + "logps/chosen": -238.91720581054688, + "logps/rejected": -265.1916198730469, + "loss": 0.5699, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.2072840929031372, + "rewards/margins": 0.3541075587272644, + "rewards/rejected": -0.5613916516304016, + "step": 3060 + }, + { + "epoch": 0.4, + "learning_rate": 3.7374510828274673e-06, + "logits/chosen": -2.516413688659668, + "logits/rejected": -2.531301498413086, + "logps/chosen": -277.7062072753906, + "logps/rejected": -348.3893127441406, + "loss": 0.6018, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19995872676372528, + "rewards/margins": 0.2855435311794281, + "rewards/rejected": -0.4855022430419922, + "step": 3070 + }, + { + "epoch": 0.4, + "learning_rate": 3.72751475454049e-06, + "logits/chosen": -2.5162839889526367, + "logits/rejected": -2.4353349208831787, + "logps/chosen": -319.59063720703125, + "logps/rejected": -318.60955810546875, + "loss": 0.5743, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.20623521506786346, + "rewards/margins": 0.3293301463127136, + "rewards/rejected": -0.5355653166770935, + "step": 3080 + }, + { + "epoch": 0.4, + "learning_rate": 3.7175528092892503e-06, + "logits/chosen": -2.6033437252044678, + "logits/rejected": -2.5976102352142334, + "logps/chosen": -297.6019592285156, + "logps/rejected": -282.84210205078125, + "loss": 0.6204, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.22910642623901367, + "rewards/margins": 0.24689094722270966, + "rewards/rejected": -0.47599735856056213, + "step": 3090 + }, + { + "epoch": 0.41, + "learning_rate": 3.7075654549692498e-06, + "logits/chosen": -2.5105113983154297, + "logits/rejected": -2.4831717014312744, + "logps/chosen": -291.5171203613281, + "logps/rejected": -290.1797790527344, + "loss": 0.614, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2578617036342621, + "rewards/margins": 0.2936268448829651, + "rewards/rejected": -0.5514885187149048, + "step": 3100 + }, + { + "epoch": 0.41, + "learning_rate": 3.697552900006249e-06, + "logits/chosen": -2.6630959510803223, + "logits/rejected": -2.620387315750122, + "logps/chosen": -306.1521301269531, + "logps/rejected": -358.87200927734375, + "loss": 0.5936, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3080160915851593, + "rewards/margins": 0.1846139132976532, + "rewards/rejected": -0.4926300048828125, + "step": 3110 + }, + { + "epoch": 0.41, + "learning_rate": 3.6875153533519244e-06, + "logits/chosen": -2.624591827392578, + "logits/rejected": -2.493398666381836, + "logps/chosen": -308.59588623046875, + "logps/rejected": -290.4993591308594, + "loss": 0.5787, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1921892762184143, + "rewards/margins": 0.33598488569259644, + "rewards/rejected": -0.5281742215156555, + "step": 3120 + }, + { + "epoch": 0.41, + "learning_rate": 3.6774530244794992e-06, + "logits/chosen": -2.6168274879455566, + "logits/rejected": -2.5857300758361816, + "logps/chosen": -343.65496826171875, + "logps/rejected": -316.0412292480469, + "loss": 0.588, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.18571509420871735, + "rewards/margins": 0.2884679436683655, + "rewards/rejected": -0.47418302297592163, + "step": 3130 + }, + { + "epoch": 0.41, + "learning_rate": 3.667366123379378e-06, + "logits/chosen": -2.5256686210632324, + "logits/rejected": -2.486649751663208, + "logps/chosen": -301.6999206542969, + "logps/rejected": -353.1916198730469, + "loss": 0.566, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.22445659339427948, + "rewards/margins": 0.341928631067276, + "rewards/rejected": -0.5663852691650391, + "step": 3140 + }, + { + "epoch": 0.41, + "learning_rate": 3.6572548605547607e-06, + "logits/chosen": -2.6104562282562256, + "logits/rejected": -2.4722983837127686, + "logps/chosen": -324.6009216308594, + "logps/rejected": -305.4631042480469, + "loss": 0.5586, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2462780922651291, + "rewards/margins": 0.3060448169708252, + "rewards/rejected": -0.5523229241371155, + "step": 3150 + }, + { + "epoch": 0.41, + "learning_rate": 3.6471194470172538e-06, + "logits/chosen": -2.751009702682495, + "logits/rejected": -2.7248828411102295, + "logps/chosen": -379.75750732421875, + "logps/rejected": -380.437744140625, + "loss": 0.5762, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.25567176938056946, + "rewards/margins": 0.29870104789733887, + "rewards/rejected": -0.5543727874755859, + "step": 3160 + }, + { + "epoch": 0.41, + "learning_rate": 3.636960094282461e-06, + "logits/chosen": -2.548859119415283, + "logits/rejected": -2.6039133071899414, + "logps/chosen": -244.54806518554688, + "logps/rejected": -294.4767150878906, + "loss": 0.5389, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.14730247855186462, + "rewards/margins": 0.3509523272514343, + "rewards/rejected": -0.49825483560562134, + "step": 3170 + }, + { + "epoch": 0.42, + "learning_rate": 3.6267770143655743e-06, + "logits/chosen": -2.550873279571533, + "logits/rejected": -2.5480148792266846, + "logps/chosen": -288.40411376953125, + "logps/rejected": -284.4498291015625, + "loss": 0.6286, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15646964311599731, + "rewards/margins": 0.17518159747123718, + "rewards/rejected": -0.3316512405872345, + "step": 3180 + }, + { + "epoch": 0.42, + "learning_rate": 3.6165704197769484e-06, + "logits/chosen": -2.5557799339294434, + "logits/rejected": -2.531587600708008, + "logps/chosen": -258.42315673828125, + "logps/rejected": -285.3637390136719, + "loss": 0.5297, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12565067410469055, + "rewards/margins": 0.4425191879272461, + "rewards/rejected": -0.568169891834259, + "step": 3190 + }, + { + "epoch": 0.42, + "learning_rate": 3.606340523517663e-06, + "logits/chosen": -2.667398452758789, + "logits/rejected": -2.635216474533081, + "logps/chosen": -324.2204895019531, + "logps/rejected": -351.78875732421875, + "loss": 0.5878, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.22811134159564972, + "rewards/margins": 0.3804187774658203, + "rewards/rejected": -0.6085301637649536, + "step": 3200 + }, + { + "epoch": 0.42, + "learning_rate": 3.5960875390750793e-06, + "logits/chosen": -2.570162057876587, + "logits/rejected": -2.485708236694336, + "logps/chosen": -303.61395263671875, + "logps/rejected": -358.18255615234375, + "loss": 0.6511, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3610957860946655, + "rewards/margins": 0.16257894039154053, + "rewards/rejected": -0.523674726486206, + "step": 3210 + }, + { + "epoch": 0.42, + "learning_rate": 3.585811680418386e-06, + "logits/chosen": -2.5814812183380127, + "logits/rejected": -2.510042667388916, + "logps/chosen": -278.59783935546875, + "logps/rejected": -262.82220458984375, + "loss": 0.5444, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.20521113276481628, + "rewards/margins": 0.4517434537410736, + "rewards/rejected": -0.6569545269012451, + "step": 3220 + }, + { + "epoch": 0.42, + "learning_rate": 3.5755131619941347e-06, + "logits/chosen": -2.6885128021240234, + "logits/rejected": -2.6287333965301514, + "logps/chosen": -350.37750244140625, + "logps/rejected": -355.7803039550781, + "loss": 0.5734, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.30031758546829224, + "rewards/margins": 0.32573914527893066, + "rewards/rejected": -0.6260567903518677, + "step": 3230 + }, + { + "epoch": 0.42, + "learning_rate": 3.565192198721759e-06, + "logits/chosen": -2.6210732460021973, + "logits/rejected": -2.5164902210235596, + "logps/chosen": -295.5282287597656, + "logps/rejected": -249.3524169921875, + "loss": 0.6068, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1660500019788742, + "rewards/margins": 0.30626028776168823, + "rewards/rejected": -0.47231030464172363, + "step": 3240 + }, + { + "epoch": 0.43, + "learning_rate": 3.5548490059890965e-06, + "logits/chosen": -2.649322509765625, + "logits/rejected": -2.6184802055358887, + "logps/chosen": -332.575927734375, + "logps/rejected": -330.59796142578125, + "loss": 0.6232, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.217446967959404, + "rewards/margins": 0.2006688416004181, + "rewards/rejected": -0.4181157946586609, + "step": 3250 + }, + { + "epoch": 0.43, + "learning_rate": 3.5444837996478903e-06, + "logits/chosen": -2.4342164993286133, + "logits/rejected": -2.4442899227142334, + "logps/chosen": -268.0245056152344, + "logps/rejected": -336.77752685546875, + "loss": 0.5982, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.19876603782176971, + "rewards/margins": 0.32545793056488037, + "rewards/rejected": -0.5242239236831665, + "step": 3260 + }, + { + "epoch": 0.43, + "learning_rate": 3.534096796009282e-06, + "logits/chosen": -2.589726686477661, + "logits/rejected": -2.63083815574646, + "logps/chosen": -257.81732177734375, + "logps/rejected": -292.96624755859375, + "loss": 0.6304, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.1578315794467926, + "rewards/margins": 0.19295494258403778, + "rewards/rejected": -0.3507865369319916, + "step": 3270 + }, + { + "epoch": 0.43, + "learning_rate": 3.5236882118393046e-06, + "logits/chosen": -2.65148663520813, + "logits/rejected": -2.5955045223236084, + "logps/chosen": -292.2409973144531, + "logps/rejected": -288.8282470703125, + "loss": 0.5885, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.17893002927303314, + "rewards/margins": 0.21064350008964539, + "rewards/rejected": -0.38957351446151733, + "step": 3280 + }, + { + "epoch": 0.43, + "learning_rate": 3.5132582643543513e-06, + "logits/chosen": -2.5080788135528564, + "logits/rejected": -2.4526593685150146, + "logps/chosen": -284.8882751464844, + "logps/rejected": -280.62945556640625, + "loss": 0.5817, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.10132601112127304, + "rewards/margins": 0.3045029044151306, + "rewards/rejected": -0.40582889318466187, + "step": 3290 + }, + { + "epoch": 0.43, + "learning_rate": 3.5028071712166456e-06, + "logits/chosen": -2.4202044010162354, + "logits/rejected": -2.429652452468872, + "logps/chosen": -287.6400451660156, + "logps/rejected": -284.93267822265625, + "loss": 0.5631, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.011086313053965569, + "rewards/margins": 0.41601577401161194, + "rewards/rejected": -0.4271020293235779, + "step": 3300 + }, + { + "epoch": 0.43, + "learning_rate": 3.4923351505297008e-06, + "logits/chosen": -2.477534294128418, + "logits/rejected": -2.572798252105713, + "logps/chosen": -309.67822265625, + "logps/rejected": -251.0945281982422, + "loss": 0.6126, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16956202685832977, + "rewards/margins": 0.2933441996574402, + "rewards/rejected": -0.46290621161460876, + "step": 3310 + }, + { + "epoch": 0.43, + "learning_rate": 3.481842420833766e-06, + "logits/chosen": -2.6691091060638428, + "logits/rejected": -2.6291580200195312, + "logps/chosen": -295.6717529296875, + "logps/rejected": -315.63372802734375, + "loss": 0.5969, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.10933154821395874, + "rewards/margins": 0.3607695698738098, + "rewards/rejected": -0.4701010584831238, + "step": 3320 + }, + { + "epoch": 0.44, + "learning_rate": 3.4713292011012645e-06, + "logits/chosen": -2.5209145545959473, + "logits/rejected": -2.5216760635375977, + "logps/chosen": -263.0899658203125, + "logps/rejected": -292.06268310546875, + "loss": 0.5841, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.037089962512254715, + "rewards/margins": 0.273418128490448, + "rewards/rejected": -0.3105081021785736, + "step": 3330 + }, + { + "epoch": 0.44, + "learning_rate": 3.4607957107322277e-06, + "logits/chosen": -2.421794891357422, + "logits/rejected": -2.4947047233581543, + "logps/chosen": -231.6426239013672, + "logps/rejected": -287.62005615234375, + "loss": 0.5952, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14515933394432068, + "rewards/margins": 0.34159886837005615, + "rewards/rejected": -0.48675817251205444, + "step": 3340 + }, + { + "epoch": 0.44, + "learning_rate": 3.4502421695497112e-06, + "logits/chosen": -2.638671398162842, + "logits/rejected": -2.59818172454834, + "logps/chosen": -315.3717956542969, + "logps/rejected": -306.5726013183594, + "loss": 0.6529, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1656460464000702, + "rewards/margins": 0.34280601143836975, + "rewards/rejected": -0.5084519982337952, + "step": 3350 + }, + { + "epoch": 0.44, + "learning_rate": 3.4396687977952137e-06, + "logits/chosen": -2.6577351093292236, + "logits/rejected": -2.5850212574005127, + "logps/chosen": -248.22463989257812, + "logps/rejected": -258.7904357910156, + "loss": 0.6046, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.17443545162677765, + "rewards/margins": 0.25829359889030457, + "rewards/rejected": -0.4327290654182434, + "step": 3360 + }, + { + "epoch": 0.44, + "learning_rate": 3.429075816124075e-06, + "logits/chosen": -2.681647300720215, + "logits/rejected": -2.596585750579834, + "logps/chosen": -400.94036865234375, + "logps/rejected": -363.10113525390625, + "loss": 0.5466, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.19438858330249786, + "rewards/margins": 0.32669034600257874, + "rewards/rejected": -0.521078884601593, + "step": 3370 + }, + { + "epoch": 0.44, + "learning_rate": 3.418463445600874e-06, + "logits/chosen": -2.681594133377075, + "logits/rejected": -2.652360200881958, + "logps/chosen": -337.20330810546875, + "logps/rejected": -276.9524841308594, + "loss": 0.6515, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.22456975281238556, + "rewards/margins": 0.1682281792163849, + "rewards/rejected": -0.39279794692993164, + "step": 3380 + }, + { + "epoch": 0.44, + "learning_rate": 3.4078319076948173e-06, + "logits/chosen": -2.5730137825012207, + "logits/rejected": -2.4797158241271973, + "logps/chosen": -296.2660827636719, + "logps/rejected": -280.090576171875, + "loss": 0.6146, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.24738606810569763, + "rewards/margins": 0.13821235299110413, + "rewards/rejected": -0.38559848070144653, + "step": 3390 + }, + { + "epoch": 0.44, + "learning_rate": 3.3971814242751123e-06, + "logits/chosen": -2.5072569847106934, + "logits/rejected": -2.531062364578247, + "logps/chosen": -335.22747802734375, + "logps/rejected": -343.5042419433594, + "loss": 0.625, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.21663913130760193, + "rewards/margins": 0.30990713834762573, + "rewards/rejected": -0.5265463590621948, + "step": 3400 + }, + { + "epoch": 0.45, + "learning_rate": 3.386512217606339e-06, + "logits/chosen": -2.6378777027130127, + "logits/rejected": -2.516758441925049, + "logps/chosen": -294.718994140625, + "logps/rejected": -314.53192138671875, + "loss": 0.5579, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.16852737963199615, + "rewards/margins": 0.29910632967948914, + "rewards/rejected": -0.4676336646080017, + "step": 3410 + }, + { + "epoch": 0.45, + "learning_rate": 3.375824510343816e-06, + "logits/chosen": -2.488013505935669, + "logits/rejected": -2.472822666168213, + "logps/chosen": -248.4076690673828, + "logps/rejected": -239.4960479736328, + "loss": 0.6288, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2241649329662323, + "rewards/margins": 0.27233701944351196, + "rewards/rejected": -0.4965019226074219, + "step": 3420 + }, + { + "epoch": 0.45, + "learning_rate": 3.3651185255289466e-06, + "logits/chosen": -2.6994948387145996, + "logits/rejected": -2.635267734527588, + "logps/chosen": -312.0728454589844, + "logps/rejected": -306.41021728515625, + "loss": 0.5684, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1855587512254715, + "rewards/margins": 0.33502134680747986, + "rewards/rejected": -0.5205801129341125, + "step": 3430 + }, + { + "epoch": 0.45, + "learning_rate": 3.354394486584568e-06, + "logits/chosen": -2.4346115589141846, + "logits/rejected": -2.4193854331970215, + "logps/chosen": -253.3272247314453, + "logps/rejected": -299.01416015625, + "loss": 0.6398, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.20227909088134766, + "rewards/margins": 0.1328783482313156, + "rewards/rejected": -0.33515748381614685, + "step": 3440 + }, + { + "epoch": 0.45, + "learning_rate": 3.3436526173102913e-06, + "logits/chosen": -2.5650057792663574, + "logits/rejected": -2.5391316413879395, + "logps/chosen": -324.17486572265625, + "logps/rejected": -323.8554382324219, + "loss": 0.59, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.16734786331653595, + "rewards/margins": 0.2571793496608734, + "rewards/rejected": -0.42452722787857056, + "step": 3450 + }, + { + "epoch": 0.45, + "learning_rate": 3.3328931418778254e-06, + "logits/chosen": -2.5991523265838623, + "logits/rejected": -2.483994722366333, + "logps/chosen": -273.0755920410156, + "logps/rejected": -265.11083984375, + "loss": 0.5952, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.25192922353744507, + "rewards/margins": 0.3073795735836029, + "rewards/rejected": -0.5593088269233704, + "step": 3460 + }, + { + "epoch": 0.45, + "learning_rate": 3.3221162848263028e-06, + "logits/chosen": -2.590855121612549, + "logits/rejected": -2.6153273582458496, + "logps/chosen": -305.8322448730469, + "logps/rejected": -262.98583984375, + "loss": 0.6158, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.30184227228164673, + "rewards/margins": 0.24508753418922424, + "rewards/rejected": -0.5469297170639038, + "step": 3470 + }, + { + "epoch": 0.46, + "learning_rate": 3.3113222710575914e-06, + "logits/chosen": -2.6684579849243164, + "logits/rejected": -2.6100895404815674, + "logps/chosen": -346.12347412109375, + "logps/rejected": -341.442626953125, + "loss": 0.6017, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.18102851510047913, + "rewards/margins": 0.3186485767364502, + "rewards/rejected": -0.49967703223228455, + "step": 3480 + }, + { + "epoch": 0.46, + "learning_rate": 3.300511325831603e-06, + "logits/chosen": -2.638643264770508, + "logits/rejected": -2.5978457927703857, + "logps/chosen": -328.1517028808594, + "logps/rejected": -293.4931335449219, + "loss": 0.5997, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.29977235198020935, + "rewards/margins": 0.23009338974952698, + "rewards/rejected": -0.5298658013343811, + "step": 3490 + }, + { + "epoch": 0.46, + "learning_rate": 3.289683674761592e-06, + "logits/chosen": -2.625931739807129, + "logits/rejected": -2.6796207427978516, + "logps/chosen": -332.6492614746094, + "logps/rejected": -315.2752990722656, + "loss": 0.5766, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.2368648499250412, + "rewards/margins": 0.351283460855484, + "rewards/rejected": -0.588148295879364, + "step": 3500 + }, + { + "epoch": 0.46, + "learning_rate": 3.2788395438094444e-06, + "logits/chosen": -2.543121814727783, + "logits/rejected": -2.523128032684326, + "logps/chosen": -298.61126708984375, + "logps/rejected": -305.8750915527344, + "loss": 0.6151, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.16757427155971527, + "rewards/margins": 0.27241280674934387, + "rewards/rejected": -0.43998709321022034, + "step": 3510 + }, + { + "epoch": 0.46, + "learning_rate": 3.2679791592809653e-06, + "logits/chosen": -2.568057060241699, + "logits/rejected": -2.665128231048584, + "logps/chosen": -282.148681640625, + "logps/rejected": -323.38238525390625, + "loss": 0.5821, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2427080124616623, + "rewards/margins": 0.2534430921077728, + "rewards/rejected": -0.4961511194705963, + "step": 3520 + }, + { + "epoch": 0.46, + "learning_rate": 3.257102747821157e-06, + "logits/chosen": -2.4993700981140137, + "logits/rejected": -2.559767246246338, + "logps/chosen": -326.0221862792969, + "logps/rejected": -322.7140197753906, + "loss": 0.6242, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.23177289962768555, + "rewards/margins": 0.330489844083786, + "rewards/rejected": -0.5622627139091492, + "step": 3530 + }, + { + "epoch": 0.46, + "learning_rate": 3.246210536409484e-06, + "logits/chosen": -2.466169834136963, + "logits/rejected": -2.5016674995422363, + "logps/chosen": -224.3962860107422, + "logps/rejected": -225.11111450195312, + "loss": 0.594, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.26115483045578003, + "rewards/margins": 0.2677431106567383, + "rewards/rejected": -0.5288979411125183, + "step": 3540 + }, + { + "epoch": 0.46, + "learning_rate": 3.235302752355142e-06, + "logits/chosen": -2.3972480297088623, + "logits/rejected": -2.4166183471679688, + "logps/chosen": -303.21630859375, + "logps/rejected": -310.1206359863281, + "loss": 0.5783, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.3191254734992981, + "rewards/margins": 0.32761865854263306, + "rewards/rejected": -0.6467441320419312, + "step": 3550 + }, + { + "epoch": 0.47, + "learning_rate": 3.2243796232923097e-06, + "logits/chosen": -2.5244884490966797, + "logits/rejected": -2.566132068634033, + "logps/chosen": -226.5001220703125, + "logps/rejected": -263.5140380859375, + "loss": 0.6309, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.17714844644069672, + "rewards/margins": 0.2616042494773865, + "rewards/rejected": -0.43875280022621155, + "step": 3560 + }, + { + "epoch": 0.47, + "learning_rate": 3.2134413771754037e-06, + "logits/chosen": -2.4819984436035156, + "logits/rejected": -2.504448890686035, + "logps/chosen": -289.71966552734375, + "logps/rejected": -281.09686279296875, + "loss": 0.561, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.19265899062156677, + "rewards/margins": 0.31390810012817383, + "rewards/rejected": -0.506567120552063, + "step": 3570 + }, + { + "epoch": 0.47, + "learning_rate": 3.2024882422743118e-06, + "logits/chosen": -2.560467481613159, + "logits/rejected": -2.511833667755127, + "logps/chosen": -282.52093505859375, + "logps/rejected": -281.93939208984375, + "loss": 0.6001, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.17843423783779144, + "rewards/margins": 0.31184864044189453, + "rewards/rejected": -0.4902828633785248, + "step": 3580 + }, + { + "epoch": 0.47, + "learning_rate": 3.1915204471696425e-06, + "logits/chosen": -2.5695555210113525, + "logits/rejected": -2.576490640640259, + "logps/chosen": -296.9548645019531, + "logps/rejected": -303.47674560546875, + "loss": 0.5785, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.22846440970897675, + "rewards/margins": 0.3821622431278229, + "rewards/rejected": -0.6106266379356384, + "step": 3590 + }, + { + "epoch": 0.47, + "learning_rate": 3.180538220747943e-06, + "logits/chosen": -2.483707904815674, + "logits/rejected": -2.5312798023223877, + "logps/chosen": -276.6914367675781, + "logps/rejected": -263.9627990722656, + "loss": 0.6374, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.33055758476257324, + "rewards/margins": 0.12794001400470734, + "rewards/rejected": -0.4584975838661194, + "step": 3600 + }, + { + "epoch": 0.47, + "learning_rate": 3.1695417921969287e-06, + "logits/chosen": -2.5744469165802, + "logits/rejected": -2.7000794410705566, + "logps/chosen": -263.73992919921875, + "logps/rejected": -256.0915832519531, + "loss": 0.569, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.22559526562690735, + "rewards/margins": 0.28088870644569397, + "rewards/rejected": -0.5064839720726013, + "step": 3610 + }, + { + "epoch": 0.47, + "learning_rate": 3.158531391000697e-06, + "logits/chosen": -2.6358370780944824, + "logits/rejected": -2.648641347885132, + "logps/chosen": -355.3272399902344, + "logps/rejected": -328.2330322265625, + "loss": 0.5619, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.030888553708791733, + "rewards/margins": 0.43781572580337524, + "rewards/rejected": -0.46870431303977966, + "step": 3620 + }, + { + "epoch": 0.48, + "learning_rate": 3.147507246934943e-06, + "logits/chosen": -2.6370327472686768, + "logits/rejected": -2.6142680644989014, + "logps/chosen": -317.4229431152344, + "logps/rejected": -301.7786560058594, + "loss": 0.5968, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14548811316490173, + "rewards/margins": 0.24573619663715363, + "rewards/rejected": -0.39122429490089417, + "step": 3630 + }, + { + "epoch": 0.48, + "learning_rate": 3.136469590062158e-06, + "logits/chosen": -2.6314265727996826, + "logits/rejected": -2.547762632369995, + "logps/chosen": -284.0455322265625, + "logps/rejected": -252.8461151123047, + "loss": 0.6125, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1615857183933258, + "rewards/margins": 0.3351583480834961, + "rewards/rejected": -0.4967440068721771, + "step": 3640 + }, + { + "epoch": 0.48, + "learning_rate": 3.1254186507268354e-06, + "logits/chosen": -2.647684097290039, + "logits/rejected": -2.5562212467193604, + "logps/chosen": -334.295654296875, + "logps/rejected": -313.12054443359375, + "loss": 0.5983, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.16967463493347168, + "rewards/margins": 0.28552836179733276, + "rewards/rejected": -0.45520296692848206, + "step": 3650 + }, + { + "epoch": 0.48, + "learning_rate": 3.114354659550656e-06, + "logits/chosen": -2.6611533164978027, + "logits/rejected": -2.6439900398254395, + "logps/chosen": -297.71160888671875, + "logps/rejected": -358.6143493652344, + "loss": 0.6145, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.2865394949913025, + "rewards/margins": 0.2464900016784668, + "rewards/rejected": -0.5330294966697693, + "step": 3660 + }, + { + "epoch": 0.48, + "learning_rate": 3.1032778474276816e-06, + "logits/chosen": -2.620631694793701, + "logits/rejected": -2.472928524017334, + "logps/chosen": -323.84246826171875, + "logps/rejected": -308.86431884765625, + "loss": 0.6706, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.393810510635376, + "rewards/margins": 0.13828083872795105, + "rewards/rejected": -0.5320913791656494, + "step": 3670 + }, + { + "epoch": 0.48, + "learning_rate": 3.092188445519532e-06, + "logits/chosen": -2.511369228363037, + "logits/rejected": -2.5086829662323, + "logps/chosen": -310.9083557128906, + "logps/rejected": -297.79791259765625, + "loss": 0.5898, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.1467629224061966, + "rewards/margins": 0.32766348123550415, + "rewards/rejected": -0.47442641854286194, + "step": 3680 + }, + { + "epoch": 0.48, + "learning_rate": 3.081086685250565e-06, + "logits/chosen": -2.709105968475342, + "logits/rejected": -2.684390068054199, + "logps/chosen": -345.8997497558594, + "logps/rejected": -322.6979064941406, + "loss": 0.5933, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.19092276692390442, + "rewards/margins": 0.25971782207489014, + "rewards/rejected": -0.4506405293941498, + "step": 3690 + }, + { + "epoch": 0.48, + "learning_rate": 3.0699727983030434e-06, + "logits/chosen": -2.693297863006592, + "logits/rejected": -2.6480326652526855, + "logps/chosen": -312.6332702636719, + "logps/rejected": -364.988525390625, + "loss": 0.6024, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1347484141588211, + "rewards/margins": 0.36063846945762634, + "rewards/rejected": -0.4953867793083191, + "step": 3700 + }, + { + "epoch": 0.49, + "learning_rate": 3.058847016612301e-06, + "logits/chosen": -2.680645704269409, + "logits/rejected": -2.5878353118896484, + "logps/chosen": -380.2493896484375, + "logps/rejected": -363.9061279296875, + "loss": 0.5614, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2745390236377716, + "rewards/margins": 0.3324805796146393, + "rewards/rejected": -0.6070197224617004, + "step": 3710 + }, + { + "epoch": 0.49, + "learning_rate": 3.0477095723619034e-06, + "logits/chosen": -2.67134952545166, + "logits/rejected": -2.5982470512390137, + "logps/chosen": -321.2911376953125, + "logps/rejected": -349.79022216796875, + "loss": 0.5855, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3027748763561249, + "rewards/margins": 0.35173699259757996, + "rewards/rejected": -0.6545118093490601, + "step": 3720 + }, + { + "epoch": 0.49, + "learning_rate": 3.0365606979788003e-06, + "logits/chosen": -2.4542784690856934, + "logits/rejected": -2.5599284172058105, + "logps/chosen": -244.4676513671875, + "logps/rejected": -282.34149169921875, + "loss": 0.6112, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.23147158324718475, + "rewards/margins": 0.22280550003051758, + "rewards/rejected": -0.4542770981788635, + "step": 3730 + }, + { + "epoch": 0.49, + "learning_rate": 3.0254006261284786e-06, + "logits/chosen": -2.676450729370117, + "logits/rejected": -2.698291301727295, + "logps/chosen": -315.92138671875, + "logps/rejected": -318.750244140625, + "loss": 0.5708, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.15017244219779968, + "rewards/margins": 0.34940075874328613, + "rewards/rejected": -0.4995731711387634, + "step": 3740 + }, + { + "epoch": 0.49, + "learning_rate": 3.0142295897101032e-06, + "logits/chosen": -2.508303165435791, + "logits/rejected": -2.5160272121429443, + "logps/chosen": -296.66790771484375, + "logps/rejected": -296.8723449707031, + "loss": 0.5772, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1724482625722885, + "rewards/margins": 0.4398605227470398, + "rewards/rejected": -0.6123087406158447, + "step": 3750 + }, + { + "epoch": 0.49, + "learning_rate": 3.0030478218516578e-06, + "logits/chosen": -2.6099557876586914, + "logits/rejected": -2.4607856273651123, + "logps/chosen": -300.81793212890625, + "logps/rejected": -226.68038940429688, + "loss": 0.5685, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1304190754890442, + "rewards/margins": 0.4726713299751282, + "rewards/rejected": -0.6030904054641724, + "step": 3760 + }, + { + "epoch": 0.49, + "learning_rate": 2.9918555559050826e-06, + "logits/chosen": -2.660377264022827, + "logits/rejected": -2.6089494228363037, + "logps/chosen": -302.4432678222656, + "logps/rejected": -351.26971435546875, + "loss": 0.6301, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2564275562763214, + "rewards/margins": 0.2971305251121521, + "rewards/rejected": -0.5535580515861511, + "step": 3770 + }, + { + "epoch": 0.49, + "learning_rate": 2.980653025441399e-06, + "logits/chosen": -2.7107796669006348, + "logits/rejected": -2.7100653648376465, + "logps/chosen": -257.9647521972656, + "logps/rejected": -319.4724426269531, + "loss": 0.6101, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2261107861995697, + "rewards/margins": 0.21425211429595947, + "rewards/rejected": -0.44036293029785156, + "step": 3780 + }, + { + "epoch": 0.5, + "learning_rate": 2.969440464245841e-06, + "logits/chosen": -2.531522274017334, + "logits/rejected": -2.5888233184814453, + "logps/chosen": -250.8273162841797, + "logps/rejected": -275.148681640625, + "loss": 0.6257, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.20485302805900574, + "rewards/margins": 0.13838718831539154, + "rewards/rejected": -0.34324023127555847, + "step": 3790 + }, + { + "epoch": 0.5, + "learning_rate": 2.95821810631297e-06, + "logits/chosen": -2.6218361854553223, + "logits/rejected": -2.605215311050415, + "logps/chosen": -330.93865966796875, + "logps/rejected": -379.01898193359375, + "loss": 0.6228, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2750341296195984, + "rewards/margins": 0.23783652484416962, + "rewards/rejected": -0.5128706693649292, + "step": 3800 + }, + { + "epoch": 0.5, + "learning_rate": 2.946986185841801e-06, + "logits/chosen": -2.5598433017730713, + "logits/rejected": -2.4834113121032715, + "logps/chosen": -294.7371520996094, + "logps/rejected": -299.9499206542969, + "loss": 0.6378, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.20022502541542053, + "rewards/margins": 0.2527645528316498, + "rewards/rejected": -0.4529895782470703, + "step": 3810 + }, + { + "epoch": 0.5, + "learning_rate": 2.935744937230903e-06, + "logits/chosen": -2.571002721786499, + "logits/rejected": -2.62736177444458, + "logps/chosen": -302.08734130859375, + "logps/rejected": -268.59735107421875, + "loss": 0.6057, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20417217910289764, + "rewards/margins": 0.22127452492713928, + "rewards/rejected": -0.4254467487335205, + "step": 3820 + }, + { + "epoch": 0.5, + "learning_rate": 2.924494595073517e-06, + "logits/chosen": -2.4782519340515137, + "logits/rejected": -2.4902045726776123, + "logps/chosen": -256.9552001953125, + "logps/rejected": -268.467041015625, + "loss": 0.5822, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12104479223489761, + "rewards/margins": 0.33370283246040344, + "rewards/rejected": -0.45474761724472046, + "step": 3830 + }, + { + "epoch": 0.5, + "learning_rate": 2.9132353941526575e-06, + "logits/chosen": -2.640956401824951, + "logits/rejected": -2.7130298614501953, + "logps/chosen": -322.9231262207031, + "logps/rejected": -281.8253479003906, + "loss": 0.584, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13268163800239563, + "rewards/margins": 0.3568686246871948, + "rewards/rejected": -0.48955026268959045, + "step": 3840 + }, + { + "epoch": 0.5, + "learning_rate": 2.901967569436209e-06, + "logits/chosen": -2.459228038787842, + "logits/rejected": -2.4970715045928955, + "logps/chosen": -276.39727783203125, + "logps/rejected": -308.2503967285156, + "loss": 0.643, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.23280692100524902, + "rewards/margins": 0.25167709589004517, + "rewards/rejected": -0.4844840168952942, + "step": 3850 + }, + { + "epoch": 0.51, + "learning_rate": 2.89069135607203e-06, + "logits/chosen": -2.556077718734741, + "logits/rejected": -2.4842946529388428, + "logps/chosen": -309.0411071777344, + "logps/rejected": -302.27496337890625, + "loss": 0.5564, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.06841972470283508, + "rewards/margins": 0.40710344910621643, + "rewards/rejected": -0.4755231440067291, + "step": 3860 + }, + { + "epoch": 0.51, + "learning_rate": 2.8794069893830386e-06, + "logits/chosen": -2.4000508785247803, + "logits/rejected": -2.368074655532837, + "logps/chosen": -318.82696533203125, + "logps/rejected": -374.33514404296875, + "loss": 0.5781, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09505738317966461, + "rewards/margins": 0.2986064553260803, + "rewards/rejected": -0.39366385340690613, + "step": 3870 + }, + { + "epoch": 0.51, + "learning_rate": 2.8681147048623038e-06, + "logits/chosen": -2.6203765869140625, + "logits/rejected": -2.622159242630005, + "logps/chosen": -339.40447998046875, + "logps/rejected": -289.11962890625, + "loss": 0.6172, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.20032520592212677, + "rewards/margins": 0.13905063271522522, + "rewards/rejected": -0.33937588334083557, + "step": 3880 + }, + { + "epoch": 0.51, + "learning_rate": 2.8568147381681333e-06, + "logits/chosen": -2.635042190551758, + "logits/rejected": -2.4665164947509766, + "logps/chosen": -334.91741943359375, + "logps/rejected": -312.35784912109375, + "loss": 0.5751, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1310148388147354, + "rewards/margins": 0.31944572925567627, + "rewards/rejected": -0.4504605829715729, + "step": 3890 + }, + { + "epoch": 0.51, + "learning_rate": 2.8455073251191533e-06, + "logits/chosen": -2.6655802726745605, + "logits/rejected": -2.5834178924560547, + "logps/chosen": -306.61749267578125, + "logps/rejected": -317.67901611328125, + "loss": 0.5782, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.17261703312397003, + "rewards/margins": 0.29314273595809937, + "rewards/rejected": -0.4657597541809082, + "step": 3900 + }, + { + "epoch": 0.51, + "learning_rate": 2.8341927016893887e-06, + "logits/chosen": -2.6398985385894775, + "logits/rejected": -2.5800163745880127, + "logps/chosen": -316.98712158203125, + "logps/rejected": -316.6258850097656, + "loss": 0.631, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.25684791803359985, + "rewards/margins": 0.24398711323738098, + "rewards/rejected": -0.5008348822593689, + "step": 3910 + }, + { + "epoch": 0.51, + "learning_rate": 2.822871104003335e-06, + "logits/chosen": -2.5508549213409424, + "logits/rejected": -2.56037974357605, + "logps/chosen": -268.0732116699219, + "logps/rejected": -326.3709411621094, + "loss": 0.5836, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20186085999011993, + "rewards/margins": 0.32339686155319214, + "rewards/rejected": -0.5252577066421509, + "step": 3920 + }, + { + "epoch": 0.51, + "learning_rate": 2.8115427683310355e-06, + "logits/chosen": -2.626038074493408, + "logits/rejected": -2.6347310543060303, + "logps/chosen": -272.03289794921875, + "logps/rejected": -298.04791259765625, + "loss": 0.5635, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1626121997833252, + "rewards/margins": 0.28776103258132935, + "rewards/rejected": -0.45037323236465454, + "step": 3930 + }, + { + "epoch": 0.52, + "learning_rate": 2.8002079310831477e-06, + "logits/chosen": -2.581829071044922, + "logits/rejected": -2.4708094596862793, + "logps/chosen": -306.98431396484375, + "logps/rejected": -275.0307922363281, + "loss": 0.589, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2808947265148163, + "rewards/margins": 0.3008837401866913, + "rewards/rejected": -0.5817785859107971, + "step": 3940 + }, + { + "epoch": 0.52, + "learning_rate": 2.7888668288060095e-06, + "logits/chosen": -2.575941562652588, + "logits/rejected": -2.555351495742798, + "logps/chosen": -281.966796875, + "logps/rejected": -320.59222412109375, + "loss": 0.5753, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.3274231255054474, + "rewards/margins": 0.3251662850379944, + "rewards/rejected": -0.6525893807411194, + "step": 3950 + }, + { + "epoch": 0.52, + "learning_rate": 2.7775196981767044e-06, + "logits/chosen": -2.67581844329834, + "logits/rejected": -2.6105871200561523, + "logps/chosen": -275.9292907714844, + "logps/rejected": -278.40631103515625, + "loss": 0.5964, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.32996195554733276, + "rewards/margins": 0.20140254497528076, + "rewards/rejected": -0.5313645601272583, + "step": 3960 + }, + { + "epoch": 0.52, + "learning_rate": 2.7661667759981213e-06, + "logits/chosen": -2.4484341144561768, + "logits/rejected": -2.5259575843811035, + "logps/chosen": -239.6707763671875, + "logps/rejected": -248.3018035888672, + "loss": 0.6396, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2669145166873932, + "rewards/margins": 0.1268802285194397, + "rewards/rejected": -0.3937947750091553, + "step": 3970 + }, + { + "epoch": 0.52, + "learning_rate": 2.7548082991940137e-06, + "logits/chosen": -2.6260974407196045, + "logits/rejected": -2.5910682678222656, + "logps/chosen": -371.46014404296875, + "logps/rejected": -330.17901611328125, + "loss": 0.5669, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2598438858985901, + "rewards/margins": 0.3069099485874176, + "rewards/rejected": -0.5667537450790405, + "step": 3980 + }, + { + "epoch": 0.52, + "learning_rate": 2.743444504804051e-06, + "logits/chosen": -2.5706257820129395, + "logits/rejected": -2.472374200820923, + "logps/chosen": -288.25201416015625, + "logps/rejected": -244.6339569091797, + "loss": 0.6152, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.19544066488742828, + "rewards/margins": 0.24673247337341309, + "rewards/rejected": -0.44217315316200256, + "step": 3990 + }, + { + "epoch": 0.52, + "learning_rate": 2.7320756299788788e-06, + "logits/chosen": -2.502800464630127, + "logits/rejected": -2.5665602684020996, + "logps/chosen": -273.9783630371094, + "logps/rejected": -309.87042236328125, + "loss": 0.5997, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.17435002326965332, + "rewards/margins": 0.3144958019256592, + "rewards/rejected": -0.4888457655906677, + "step": 4000 + }, + { + "epoch": 0.52, + "learning_rate": 2.7207019119751644e-06, + "logits/chosen": -2.469245195388794, + "logits/rejected": -2.538689613342285, + "logps/chosen": -318.72247314453125, + "logps/rejected": -307.48931884765625, + "loss": 0.5864, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.19669604301452637, + "rewards/margins": 0.3518393635749817, + "rewards/rejected": -0.5485354661941528, + "step": 4010 + }, + { + "epoch": 0.53, + "learning_rate": 2.7093235881506474e-06, + "logits/chosen": -2.592909812927246, + "logits/rejected": -2.5366950035095215, + "logps/chosen": -321.53375244140625, + "logps/rejected": -306.61614990234375, + "loss": 0.5879, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.30671948194503784, + "rewards/margins": 0.25378814339637756, + "rewards/rejected": -0.5605076551437378, + "step": 4020 + }, + { + "epoch": 0.53, + "learning_rate": 2.6979408959591863e-06, + "logits/chosen": -2.5961556434631348, + "logits/rejected": -2.5295894145965576, + "logps/chosen": -316.2151184082031, + "logps/rejected": -316.78057861328125, + "loss": 0.5385, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.21914584934711456, + "rewards/margins": 0.5072210431098938, + "rewards/rejected": -0.7263668179512024, + "step": 4030 + }, + { + "epoch": 0.53, + "learning_rate": 2.6865540729458034e-06, + "logits/chosen": -2.607487440109253, + "logits/rejected": -2.5697755813598633, + "logps/chosen": -334.29986572265625, + "logps/rejected": -336.15380859375, + "loss": 0.552, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.19254732131958008, + "rewards/margins": 0.26985567808151245, + "rewards/rejected": -0.46240299940109253, + "step": 4040 + }, + { + "epoch": 0.53, + "learning_rate": 2.675163356741726e-06, + "logits/chosen": -2.5060925483703613, + "logits/rejected": -2.631974697113037, + "logps/chosen": -270.8483581542969, + "logps/rejected": -266.566650390625, + "loss": 0.542, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.05409348011016846, + "rewards/margins": 0.4574395716190338, + "rewards/rejected": -0.5115330815315247, + "step": 4050 + }, + { + "epoch": 0.53, + "learning_rate": 2.6637689850594285e-06, + "logits/chosen": -2.318814754486084, + "logits/rejected": -2.3529763221740723, + "logps/chosen": -351.1209716796875, + "logps/rejected": -386.14630126953125, + "loss": 0.5833, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.30523282289505005, + "rewards/margins": 0.37871304154396057, + "rewards/rejected": -0.683945894241333, + "step": 4060 + }, + { + "epoch": 0.53, + "learning_rate": 2.652371195687671e-06, + "logits/chosen": -2.5676889419555664, + "logits/rejected": -2.596935749053955, + "logps/chosen": -326.96649169921875, + "logps/rejected": -374.5132141113281, + "loss": 0.573, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.23799224197864532, + "rewards/margins": 0.4420638084411621, + "rewards/rejected": -0.6800560355186462, + "step": 4070 + }, + { + "epoch": 0.53, + "learning_rate": 2.64097022648654e-06, + "logits/chosen": -2.6802048683166504, + "logits/rejected": -2.548435926437378, + "logps/chosen": -307.8295593261719, + "logps/rejected": -279.1652526855469, + "loss": 0.6306, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.32306724786758423, + "rewards/margins": 0.1645546853542328, + "rewards/rejected": -0.487621933221817, + "step": 4080 + }, + { + "epoch": 0.54, + "learning_rate": 2.6295663153824774e-06, + "logits/chosen": -2.7688498497009277, + "logits/rejected": -2.663788080215454, + "logps/chosen": -351.2892761230469, + "logps/rejected": -305.7450256347656, + "loss": 0.5699, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.27330145239830017, + "rewards/margins": 0.373261421918869, + "rewards/rejected": -0.646562933921814, + "step": 4090 + }, + { + "epoch": 0.54, + "learning_rate": 2.6181597003633218e-06, + "logits/chosen": -2.6223292350769043, + "logits/rejected": -2.60467529296875, + "logps/chosen": -251.3638916015625, + "logps/rejected": -261.7774963378906, + "loss": 0.6003, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.21685147285461426, + "rewards/margins": 0.2799449563026428, + "rewards/rejected": -0.4967964291572571, + "step": 4100 + }, + { + "epoch": 0.54, + "learning_rate": 2.606750619473342e-06, + "logits/chosen": -2.49078106880188, + "logits/rejected": -2.4936556816101074, + "logps/chosen": -250.99197387695312, + "logps/rejected": -311.34246826171875, + "loss": 0.582, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1258927881717682, + "rewards/margins": 0.3767452836036682, + "rewards/rejected": -0.502638041973114, + "step": 4110 + }, + { + "epoch": 0.54, + "learning_rate": 2.595339310808262e-06, + "logits/chosen": -2.5719306468963623, + "logits/rejected": -2.5432655811309814, + "logps/chosen": -330.5278015136719, + "logps/rejected": -290.69866943359375, + "loss": 0.5878, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13885828852653503, + "rewards/margins": 0.3301781117916107, + "rewards/rejected": -0.46903640031814575, + "step": 4120 + }, + { + "epoch": 0.54, + "learning_rate": 2.5839260125103004e-06, + "logits/chosen": -2.430999517440796, + "logits/rejected": -2.393784761428833, + "logps/chosen": -207.4061737060547, + "logps/rejected": -249.16732788085938, + "loss": 0.5656, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.24477513134479523, + "rewards/margins": 0.2816818654537201, + "rewards/rejected": -0.5264570116996765, + "step": 4130 + }, + { + "epoch": 0.54, + "learning_rate": 2.5725109627631984e-06, + "logits/chosen": -2.6551685333251953, + "logits/rejected": -2.621466636657715, + "logps/chosen": -310.831787109375, + "logps/rejected": -279.20166015625, + "loss": 0.6171, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.20539292693138123, + "rewards/margins": 0.2558387517929077, + "rewards/rejected": -0.46123170852661133, + "step": 4140 + }, + { + "epoch": 0.54, + "learning_rate": 2.5610943997872443e-06, + "logits/chosen": -2.5449509620666504, + "logits/rejected": -2.489164352416992, + "logps/chosen": -268.4985656738281, + "logps/rejected": -314.49078369140625, + "loss": 0.6003, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2852312922477722, + "rewards/margins": 0.25852587819099426, + "rewards/rejected": -0.5437571406364441, + "step": 4150 + }, + { + "epoch": 0.54, + "learning_rate": 2.5496765618343096e-06, + "logits/chosen": -2.5548152923583984, + "logits/rejected": -2.5760722160339355, + "logps/chosen": -292.0989990234375, + "logps/rejected": -324.4699401855469, + "loss": 0.4914, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.10181447118520737, + "rewards/margins": 0.6143667101860046, + "rewards/rejected": -0.7161811590194702, + "step": 4160 + }, + { + "epoch": 0.55, + "learning_rate": 2.538257687182871e-06, + "logits/chosen": -2.674144744873047, + "logits/rejected": -2.5367748737335205, + "logps/chosen": -287.7279968261719, + "logps/rejected": -282.7435607910156, + "loss": 0.556, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2096879482269287, + "rewards/margins": 0.3056090772151947, + "rewards/rejected": -0.515296995639801, + "step": 4170 + }, + { + "epoch": 0.55, + "learning_rate": 2.526838014133041e-06, + "logits/chosen": -2.5615897178649902, + "logits/rejected": -2.5196616649627686, + "logps/chosen": -296.09625244140625, + "logps/rejected": -345.0995178222656, + "loss": 0.5686, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15199770033359528, + "rewards/margins": 0.41160517930984497, + "rewards/rejected": -0.5636029243469238, + "step": 4180 + }, + { + "epoch": 0.55, + "learning_rate": 2.515417781001594e-06, + "logits/chosen": -2.6643834114074707, + "logits/rejected": -2.660529851913452, + "logps/chosen": -329.8970642089844, + "logps/rejected": -324.8268127441406, + "loss": 0.6113, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.19869431853294373, + "rewards/margins": 0.2704014182090759, + "rewards/rejected": -0.46909570693969727, + "step": 4190 + }, + { + "epoch": 0.55, + "learning_rate": 2.503997226116992e-06, + "logits/chosen": -2.6225388050079346, + "logits/rejected": -2.5563571453094482, + "logps/chosen": -337.92828369140625, + "logps/rejected": -320.9072265625, + "loss": 0.5509, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.2428453415632248, + "rewards/margins": 0.45858412981033325, + "rewards/rejected": -0.7014294862747192, + "step": 4200 + }, + { + "epoch": 0.55, + "learning_rate": 2.4925765878144115e-06, + "logits/chosen": -2.5668654441833496, + "logits/rejected": -2.444279193878174, + "logps/chosen": -268.70355224609375, + "logps/rejected": -360.464599609375, + "loss": 0.5916, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.24730440974235535, + "rewards/margins": 0.29829341173171997, + "rewards/rejected": -0.5455979108810425, + "step": 4210 + }, + { + "epoch": 0.55, + "learning_rate": 2.4811561044307727e-06, + "logits/chosen": -2.598015308380127, + "logits/rejected": -2.473515033721924, + "logps/chosen": -314.43084716796875, + "logps/rejected": -324.11981201171875, + "loss": 0.5587, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16435906291007996, + "rewards/margins": 0.4245055317878723, + "rewards/rejected": -0.5888645052909851, + "step": 4220 + }, + { + "epoch": 0.55, + "learning_rate": 2.469736014299758e-06, + "logits/chosen": -2.6424834728240967, + "logits/rejected": -2.5358104705810547, + "logps/chosen": -342.76776123046875, + "logps/rejected": -308.48419189453125, + "loss": 0.5953, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.18019364774227142, + "rewards/margins": 0.3455636501312256, + "rewards/rejected": -0.5257573127746582, + "step": 4230 + }, + { + "epoch": 0.55, + "learning_rate": 2.458316555746846e-06, + "logits/chosen": -2.5592801570892334, + "logits/rejected": -2.50714373588562, + "logps/chosen": -283.90289306640625, + "logps/rejected": -320.06597900390625, + "loss": 0.5762, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09222878515720367, + "rewards/margins": 0.32841163873672485, + "rewards/rejected": -0.42064040899276733, + "step": 4240 + }, + { + "epoch": 0.56, + "learning_rate": 2.446897967084334e-06, + "logits/chosen": -2.55417537689209, + "logits/rejected": -2.5206234455108643, + "logps/chosen": -304.2807312011719, + "logps/rejected": -322.7982482910156, + "loss": 0.6128, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.13540779054164886, + "rewards/margins": 0.32829588651657104, + "rewards/rejected": -0.4637036919593811, + "step": 4250 + }, + { + "epoch": 0.56, + "learning_rate": 2.4354804866063684e-06, + "logits/chosen": -2.6554081439971924, + "logits/rejected": -2.5631651878356934, + "logps/chosen": -298.30670166015625, + "logps/rejected": -310.69866943359375, + "loss": 0.5985, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1584414690732956, + "rewards/margins": 0.32091015577316284, + "rewards/rejected": -0.479351669549942, + "step": 4260 + }, + { + "epoch": 0.56, + "learning_rate": 2.424064352583964e-06, + "logits/chosen": -2.586653232574463, + "logits/rejected": -2.4521918296813965, + "logps/chosen": -332.4245300292969, + "logps/rejected": -311.0771484375, + "loss": 0.5617, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.17719905078411102, + "rewards/margins": 0.3603389859199524, + "rewards/rejected": -0.5375381112098694, + "step": 4270 + }, + { + "epoch": 0.56, + "learning_rate": 2.4126498032600403e-06, + "logits/chosen": -2.5424716472625732, + "logits/rejected": -2.579455614089966, + "logps/chosen": -248.1239471435547, + "logps/rejected": -285.8912048339844, + "loss": 0.5824, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.24778568744659424, + "rewards/margins": 0.2968307137489319, + "rewards/rejected": -0.5446164011955261, + "step": 4280 + }, + { + "epoch": 0.56, + "learning_rate": 2.401237076844445e-06, + "logits/chosen": -2.4384758472442627, + "logits/rejected": -2.4419591426849365, + "logps/chosen": -309.3309631347656, + "logps/rejected": -324.6368713378906, + "loss": 0.6007, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.24136705696582794, + "rewards/margins": 0.2990710437297821, + "rewards/rejected": -0.5404380559921265, + "step": 4290 + }, + { + "epoch": 0.56, + "learning_rate": 2.38982641150898e-06, + "logits/chosen": -2.574749231338501, + "logits/rejected": -2.4584412574768066, + "logps/chosen": -316.0588073730469, + "logps/rejected": -311.87408447265625, + "loss": 0.5542, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.18666191399097443, + "rewards/margins": 0.36896970868110657, + "rewards/rejected": -0.5556316375732422, + "step": 4300 + }, + { + "epoch": 0.56, + "learning_rate": 2.3784180453824414e-06, + "logits/chosen": -2.554856061935425, + "logits/rejected": -2.616472005844116, + "logps/chosen": -292.73944091796875, + "logps/rejected": -310.92938232421875, + "loss": 0.5842, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2169373482465744, + "rewards/margins": 0.3188151717185974, + "rewards/rejected": -0.535752534866333, + "step": 4310 + }, + { + "epoch": 0.57, + "learning_rate": 2.367012216545638e-06, + "logits/chosen": -2.7256081104278564, + "logits/rejected": -2.644930601119995, + "logps/chosen": -323.84515380859375, + "logps/rejected": -291.82061767578125, + "loss": 0.5909, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.10996238887310028, + "rewards/margins": 0.3115832209587097, + "rewards/rejected": -0.4215455949306488, + "step": 4320 + }, + { + "epoch": 0.57, + "learning_rate": 2.3556091630264294e-06, + "logits/chosen": -2.5826218128204346, + "logits/rejected": -2.5682921409606934, + "logps/chosen": -319.13128662109375, + "logps/rejected": -362.3716125488281, + "loss": 0.544, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2128753364086151, + "rewards/margins": 0.38813719153404236, + "rewards/rejected": -0.6010125875473022, + "step": 4330 + }, + { + "epoch": 0.57, + "learning_rate": 2.344209122794757e-06, + "logits/chosen": -2.5008184909820557, + "logits/rejected": -2.4876294136047363, + "logps/chosen": -320.45379638671875, + "logps/rejected": -331.88226318359375, + "loss": 0.5932, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.2762475609779358, + "rewards/margins": 0.3098161220550537, + "rewards/rejected": -0.5860637426376343, + "step": 4340 + }, + { + "epoch": 0.57, + "learning_rate": 2.3328123337576787e-06, + "logits/chosen": -2.6126558780670166, + "logits/rejected": -2.5322470664978027, + "logps/chosen": -302.9745178222656, + "logps/rejected": -350.0872802734375, + "loss": 0.6005, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.24269449710845947, + "rewards/margins": 0.32287895679473877, + "rewards/rejected": -0.565573513507843, + "step": 4350 + }, + { + "epoch": 0.57, + "learning_rate": 2.3214190337544017e-06, + "logits/chosen": -2.5594842433929443, + "logits/rejected": -2.561929941177368, + "logps/chosen": -245.7132568359375, + "logps/rejected": -274.37982177734375, + "loss": 0.5811, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.1834389567375183, + "rewards/margins": 0.3247998356819153, + "rewards/rejected": -0.5082387328147888, + "step": 4360 + }, + { + "epoch": 0.57, + "learning_rate": 2.310029460551323e-06, + "logits/chosen": -2.6649985313415527, + "logits/rejected": -2.5254311561584473, + "logps/chosen": -293.27239990234375, + "logps/rejected": -306.5220642089844, + "loss": 0.5821, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.14221002161502838, + "rewards/margins": 0.25821369886398315, + "rewards/rejected": -0.4004237651824951, + "step": 4370 + }, + { + "epoch": 0.57, + "learning_rate": 2.2986438518370645e-06, + "logits/chosen": -2.477169990539551, + "logits/rejected": -2.546952486038208, + "logps/chosen": -269.667236328125, + "logps/rejected": -292.5154113769531, + "loss": 0.5845, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1302667260169983, + "rewards/margins": 0.3379213809967041, + "rewards/rejected": -0.4681881070137024, + "step": 4380 + }, + { + "epoch": 0.57, + "learning_rate": 2.2872624452175123e-06, + "logits/chosen": -2.5775961875915527, + "logits/rejected": -2.598184108734131, + "logps/chosen": -293.9779357910156, + "logps/rejected": -306.27972412109375, + "loss": 0.6237, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1786755621433258, + "rewards/margins": 0.22471539676189423, + "rewards/rejected": -0.40339094400405884, + "step": 4390 + }, + { + "epoch": 0.58, + "learning_rate": 2.2758854782108584e-06, + "logits/chosen": -2.5487232208251953, + "logits/rejected": -2.5394325256347656, + "logps/chosen": -273.4031066894531, + "logps/rejected": -308.8743896484375, + "loss": 0.6033, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2166653573513031, + "rewards/margins": 0.2811164855957031, + "rewards/rejected": -0.49778181314468384, + "step": 4400 + }, + { + "epoch": 0.58, + "learning_rate": 2.2645131882426458e-06, + "logits/chosen": -2.589221954345703, + "logits/rejected": -2.563668727874756, + "logps/chosen": -310.12298583984375, + "logps/rejected": -232.107177734375, + "loss": 0.6248, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13823625445365906, + "rewards/margins": 0.3391169607639313, + "rewards/rejected": -0.47735318541526794, + "step": 4410 + }, + { + "epoch": 0.58, + "learning_rate": 2.2531458126408154e-06, + "logits/chosen": -2.642242193222046, + "logits/rejected": -2.566376209259033, + "logps/chosen": -288.0643615722656, + "logps/rejected": -283.2187805175781, + "loss": 0.5621, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.18885095417499542, + "rewards/margins": 0.4437170922756195, + "rewards/rejected": -0.6325680017471313, + "step": 4420 + }, + { + "epoch": 0.58, + "learning_rate": 2.2417835886307452e-06, + "logits/chosen": -2.405334711074829, + "logits/rejected": -2.3908848762512207, + "logps/chosen": -293.4359436035156, + "logps/rejected": -289.71917724609375, + "loss": 0.561, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13935764133930206, + "rewards/margins": 0.4529578685760498, + "rewards/rejected": -0.5923154950141907, + "step": 4430 + }, + { + "epoch": 0.58, + "learning_rate": 2.2304267533303075e-06, + "logits/chosen": -2.5580015182495117, + "logits/rejected": -2.4934747219085693, + "logps/chosen": -393.26812744140625, + "logps/rejected": -347.17547607421875, + "loss": 0.6359, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.26866209506988525, + "rewards/margins": 0.199972465634346, + "rewards/rejected": -0.4686345160007477, + "step": 4440 + }, + { + "epoch": 0.58, + "learning_rate": 2.219075543744918e-06, + "logits/chosen": -2.5444393157958984, + "logits/rejected": -2.4769625663757324, + "logps/chosen": -371.289794921875, + "logps/rejected": -325.30242919921875, + "loss": 0.6073, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.19787821173667908, + "rewards/margins": 0.2811738848686218, + "rewards/rejected": -0.4790521562099457, + "step": 4450 + }, + { + "epoch": 0.58, + "learning_rate": 2.207730196762589e-06, + "logits/chosen": -2.571669340133667, + "logits/rejected": -2.502657175064087, + "logps/chosen": -299.11480712890625, + "logps/rejected": -299.9048767089844, + "loss": 0.5921, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.17682352662086487, + "rewards/margins": 0.4167998731136322, + "rewards/rejected": -0.5936234593391418, + "step": 4460 + }, + { + "epoch": 0.58, + "learning_rate": 2.1963909491489846e-06, + "logits/chosen": -2.601297616958618, + "logits/rejected": -2.589207649230957, + "logps/chosen": -237.0451202392578, + "logps/rejected": -242.80563354492188, + "loss": 0.5888, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.18458524346351624, + "rewards/margins": 0.26062318682670593, + "rewards/rejected": -0.4452084004878998, + "step": 4470 + }, + { + "epoch": 0.59, + "learning_rate": 2.185058037542486e-06, + "logits/chosen": -2.523491382598877, + "logits/rejected": -2.50538969039917, + "logps/chosen": -334.57525634765625, + "logps/rejected": -322.3340148925781, + "loss": 0.5553, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.20119798183441162, + "rewards/margins": 0.4498375356197357, + "rewards/rejected": -0.6510355472564697, + "step": 4480 + }, + { + "epoch": 0.59, + "learning_rate": 2.173731698449244e-06, + "logits/chosen": -2.4815447330474854, + "logits/rejected": -2.400696277618408, + "logps/chosen": -344.4715881347656, + "logps/rejected": -311.84320068359375, + "loss": 0.5858, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.17006592452526093, + "rewards/margins": 0.2964300215244293, + "rewards/rejected": -0.46649593114852905, + "step": 4490 + }, + { + "epoch": 0.59, + "learning_rate": 2.1624121682382495e-06, + "logits/chosen": -2.4832749366760254, + "logits/rejected": -2.4193451404571533, + "logps/chosen": -273.87847900390625, + "logps/rejected": -334.41436767578125, + "loss": 0.5469, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.20237097144126892, + "rewards/margins": 0.33962470293045044, + "rewards/rejected": -0.541995644569397, + "step": 4500 + }, + { + "epoch": 0.59, + "learning_rate": 2.1510996831363993e-06, + "logits/chosen": -2.645885944366455, + "logits/rejected": -2.535335063934326, + "logps/chosen": -345.0766906738281, + "logps/rejected": -382.795654296875, + "loss": 0.6197, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.26461997628211975, + "rewards/margins": 0.2347315549850464, + "rewards/rejected": -0.4993515908718109, + "step": 4510 + }, + { + "epoch": 0.59, + "learning_rate": 2.139794479223565e-06, + "logits/chosen": -2.5707051753997803, + "logits/rejected": -2.5140109062194824, + "logps/chosen": -303.35137939453125, + "logps/rejected": -307.03851318359375, + "loss": 0.6228, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.2665776312351227, + "rewards/margins": 0.2199481725692749, + "rewards/rejected": -0.48652583360671997, + "step": 4520 + }, + { + "epoch": 0.59, + "learning_rate": 2.128496792427669e-06, + "logits/chosen": -2.678926944732666, + "logits/rejected": -2.627936840057373, + "logps/chosen": -313.7235412597656, + "logps/rejected": -349.46246337890625, + "loss": 0.5948, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18646176159381866, + "rewards/margins": 0.2873113453388214, + "rewards/rejected": -0.4737730920314789, + "step": 4530 + }, + { + "epoch": 0.59, + "learning_rate": 2.117206858519758e-06, + "logits/chosen": -2.635826349258423, + "logits/rejected": -2.6556289196014404, + "logps/chosen": -328.98748779296875, + "logps/rejected": -319.97430419921875, + "loss": 0.6051, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.13268058001995087, + "rewards/margins": 0.26753780245780945, + "rewards/rejected": -0.4002183973789215, + "step": 4540 + }, + { + "epoch": 0.6, + "learning_rate": 2.1059249131090844e-06, + "logits/chosen": -2.457590341567993, + "logits/rejected": -2.4332821369171143, + "logps/chosen": -250.3773956298828, + "logps/rejected": -240.7942352294922, + "loss": 0.602, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.19220121204853058, + "rewards/margins": 0.16399028897285461, + "rewards/rejected": -0.356191486120224, + "step": 4550 + }, + { + "epoch": 0.6, + "learning_rate": 2.094651191638189e-06, + "logits/chosen": -2.625013828277588, + "logits/rejected": -2.550257444381714, + "logps/chosen": -319.27130126953125, + "logps/rejected": -304.85931396484375, + "loss": 0.5871, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21907266974449158, + "rewards/margins": 0.3216410279273987, + "rewards/rejected": -0.5407137274742126, + "step": 4560 + }, + { + "epoch": 0.6, + "learning_rate": 2.0833859293779867e-06, + "logits/chosen": -2.594148635864258, + "logits/rejected": -2.5711936950683594, + "logps/chosen": -306.7306823730469, + "logps/rejected": -315.5806884765625, + "loss": 0.5735, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14544863998889923, + "rewards/margins": 0.29394346475601196, + "rewards/rejected": -0.4393920302391052, + "step": 4570 + }, + { + "epoch": 0.6, + "learning_rate": 2.0721293614228568e-06, + "logits/chosen": -2.5141139030456543, + "logits/rejected": -2.489750385284424, + "logps/chosen": -298.4672546386719, + "logps/rejected": -302.94805908203125, + "loss": 0.5933, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2640727162361145, + "rewards/margins": 0.13427096605300903, + "rewards/rejected": -0.39834368228912354, + "step": 4580 + }, + { + "epoch": 0.6, + "learning_rate": 2.060881722685742e-06, + "logits/chosen": -2.4600939750671387, + "logits/rejected": -2.37214994430542, + "logps/chosen": -319.441650390625, + "logps/rejected": -313.952880859375, + "loss": 0.5497, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1119556650519371, + "rewards/margins": 0.43312448263168335, + "rewards/rejected": -0.5450801849365234, + "step": 4590 + }, + { + "epoch": 0.6, + "learning_rate": 2.049643247893235e-06, + "logits/chosen": -2.7086758613586426, + "logits/rejected": -2.5885002613067627, + "logps/chosen": -354.56170654296875, + "logps/rejected": -338.67431640625, + "loss": 0.5966, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.19458802044391632, + "rewards/margins": 0.35264259576797485, + "rewards/rejected": -0.54723060131073, + "step": 4600 + }, + { + "epoch": 0.6, + "learning_rate": 2.0384141715806903e-06, + "logits/chosen": -2.538681983947754, + "logits/rejected": -2.620173931121826, + "logps/chosen": -272.54425048828125, + "logps/rejected": -302.4497985839844, + "loss": 0.6154, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1432553231716156, + "rewards/margins": 0.21607744693756104, + "rewards/rejected": -0.35933274030685425, + "step": 4610 + }, + { + "epoch": 0.6, + "learning_rate": 2.0271947280873255e-06, + "logits/chosen": -2.521843433380127, + "logits/rejected": -2.465176582336426, + "logps/chosen": -216.7253875732422, + "logps/rejected": -281.3014221191406, + "loss": 0.6147, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.09545087814331055, + "rewards/margins": 0.41773247718811035, + "rewards/rejected": -0.5131834149360657, + "step": 4620 + }, + { + "epoch": 0.61, + "learning_rate": 2.0159851515513302e-06, + "logits/chosen": -2.434443950653076, + "logits/rejected": -2.464874744415283, + "logps/chosen": -273.6073303222656, + "logps/rejected": -292.5726013183594, + "loss": 0.5812, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2675468325614929, + "rewards/margins": 0.30386072397232056, + "rewards/rejected": -0.5714076161384583, + "step": 4630 + }, + { + "epoch": 0.61, + "learning_rate": 2.004785675904982e-06, + "logits/chosen": -2.4416751861572266, + "logits/rejected": -2.5264148712158203, + "logps/chosen": -297.03271484375, + "logps/rejected": -301.3223876953125, + "loss": 0.593, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2851831614971161, + "rewards/margins": 0.32416126132011414, + "rewards/rejected": -0.6093443632125854, + "step": 4640 + }, + { + "epoch": 0.61, + "learning_rate": 1.9935965348697624e-06, + "logits/chosen": -2.518084764480591, + "logits/rejected": -2.5146846771240234, + "logps/chosen": -279.87762451171875, + "logps/rejected": -327.8019714355469, + "loss": 0.6034, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.20778203010559082, + "rewards/margins": 0.26329201459884644, + "rewards/rejected": -0.47107404470443726, + "step": 4650 + }, + { + "epoch": 0.61, + "learning_rate": 1.9824179619514807e-06, + "logits/chosen": -2.521066188812256, + "logits/rejected": -2.5882599353790283, + "logps/chosen": -185.96694946289062, + "logps/rejected": -223.3569793701172, + "loss": 0.5953, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.09492836892604828, + "rewards/margins": 0.2776508033275604, + "rewards/rejected": -0.3725792169570923, + "step": 4660 + }, + { + "epoch": 0.61, + "learning_rate": 1.9712501904354004e-06, + "logits/chosen": -2.577836513519287, + "logits/rejected": -2.4661128520965576, + "logps/chosen": -274.2032470703125, + "logps/rejected": -246.4644012451172, + "loss": 0.6235, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2577691972255707, + "rewards/margins": 0.20414555072784424, + "rewards/rejected": -0.4619147777557373, + "step": 4670 + }, + { + "epoch": 0.61, + "learning_rate": 1.960093453381369e-06, + "logits/chosen": -2.4722354412078857, + "logits/rejected": -2.5045969486236572, + "logps/chosen": -254.60546875, + "logps/rejected": -283.6824951171875, + "loss": 0.605, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14864405989646912, + "rewards/margins": 0.29489949345588684, + "rewards/rejected": -0.44354352355003357, + "step": 4680 + }, + { + "epoch": 0.61, + "learning_rate": 1.948947983618962e-06, + "logits/chosen": -2.572479248046875, + "logits/rejected": -2.542299270629883, + "logps/chosen": -325.290283203125, + "logps/rejected": -301.44476318359375, + "loss": 0.6139, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20716340839862823, + "rewards/margins": 0.2456679344177246, + "rewards/rejected": -0.45283135771751404, + "step": 4690 + }, + { + "epoch": 0.62, + "learning_rate": 1.937814013742611e-06, + "logits/chosen": -2.552811622619629, + "logits/rejected": -2.507504940032959, + "logps/chosen": -395.3104553222656, + "logps/rejected": -375.67572021484375, + "loss": 0.6076, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.27121731638908386, + "rewards/margins": 0.22323930263519287, + "rewards/rejected": -0.49445658922195435, + "step": 4700 + }, + { + "epoch": 0.62, + "learning_rate": 1.9266917761067617e-06, + "logits/chosen": -2.515617847442627, + "logits/rejected": -2.471935749053955, + "logps/chosen": -309.8771667480469, + "logps/rejected": -280.6637878417969, + "loss": 0.581, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.24826793372631073, + "rewards/margins": 0.3144586980342865, + "rewards/rejected": -0.5627266764640808, + "step": 4710 + }, + { + "epoch": 0.62, + "learning_rate": 1.915581502821017e-06, + "logits/chosen": -2.4324276447296143, + "logits/rejected": -2.3578710556030273, + "logps/chosen": -307.38092041015625, + "logps/rejected": -407.55059814453125, + "loss": 0.5642, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2369718849658966, + "rewards/margins": 0.39206990599632263, + "rewards/rejected": -0.6290417909622192, + "step": 4720 + }, + { + "epoch": 0.62, + "learning_rate": 1.9044834257452997e-06, + "logits/chosen": -2.588669538497925, + "logits/rejected": -2.521408796310425, + "logps/chosen": -289.35467529296875, + "logps/rejected": -281.88519287109375, + "loss": 0.5973, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.13194029033184052, + "rewards/margins": 0.21657545864582062, + "rewards/rejected": -0.3485157787799835, + "step": 4730 + }, + { + "epoch": 0.62, + "learning_rate": 1.893397776485006e-06, + "logits/chosen": -2.596656084060669, + "logits/rejected": -2.514173984527588, + "logps/chosen": -304.2091369628906, + "logps/rejected": -317.41107177734375, + "loss": 0.5589, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1646658480167389, + "rewards/margins": 0.4921610951423645, + "rewards/rejected": -0.6568268537521362, + "step": 4740 + }, + { + "epoch": 0.62, + "learning_rate": 1.8823247863861804e-06, + "logits/chosen": -2.598585605621338, + "logits/rejected": -2.5877842903137207, + "logps/chosen": -315.4193420410156, + "logps/rejected": -328.97784423828125, + "loss": 0.6086, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2035294473171234, + "rewards/margins": 0.2403070479631424, + "rewards/rejected": -0.4438364505767822, + "step": 4750 + }, + { + "epoch": 0.62, + "learning_rate": 1.8712646865306822e-06, + "logits/chosen": -2.604114532470703, + "logits/rejected": -2.5948071479797363, + "logps/chosen": -368.4901123046875, + "logps/rejected": -339.9342956542969, + "loss": 0.581, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.22862637042999268, + "rewards/margins": 0.3091500699520111, + "rewards/rejected": -0.5377764701843262, + "step": 4760 + }, + { + "epoch": 0.62, + "learning_rate": 1.8602177077313631e-06, + "logits/chosen": -2.5544817447662354, + "logits/rejected": -2.4318108558654785, + "logps/chosen": -290.60748291015625, + "logps/rejected": -346.29559326171875, + "loss": 0.596, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.17622289061546326, + "rewards/margins": 0.2590067982673645, + "rewards/rejected": -0.43522968888282776, + "step": 4770 + }, + { + "epoch": 0.63, + "learning_rate": 1.8491840805272546e-06, + "logits/chosen": -2.5918362140655518, + "logits/rejected": -2.5588793754577637, + "logps/chosen": -311.6506652832031, + "logps/rejected": -301.0184631347656, + "loss": 0.6, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2586202025413513, + "rewards/margins": 0.17031022906303406, + "rewards/rejected": -0.4289304316043854, + "step": 4780 + }, + { + "epoch": 0.63, + "learning_rate": 1.8381640351787516e-06, + "logits/chosen": -2.4994304180145264, + "logits/rejected": -2.529038429260254, + "logps/chosen": -275.3616638183594, + "logps/rejected": -320.9551696777344, + "loss": 0.5659, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.10712920129299164, + "rewards/margins": 0.3530879318714142, + "rewards/rejected": -0.460217148065567, + "step": 4790 + }, + { + "epoch": 0.63, + "learning_rate": 1.8271578016628122e-06, + "logits/chosen": -2.506784677505493, + "logits/rejected": -2.5386555194854736, + "logps/chosen": -275.3206481933594, + "logps/rejected": -279.49774169921875, + "loss": 0.6083, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.22943325340747833, + "rewards/margins": 0.12677840888500214, + "rewards/rejected": -0.35621166229248047, + "step": 4800 + }, + { + "epoch": 0.63, + "learning_rate": 1.8161656096681546e-06, + "logits/chosen": -2.5047996044158936, + "logits/rejected": -2.439415454864502, + "logps/chosen": -243.07089233398438, + "logps/rejected": -295.4671325683594, + "loss": 0.5634, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.12545737624168396, + "rewards/margins": 0.5444375276565552, + "rewards/rejected": -0.669894814491272, + "step": 4810 + }, + { + "epoch": 0.63, + "learning_rate": 1.8051876885904645e-06, + "logits/chosen": -2.4755449295043945, + "logits/rejected": -2.517735004425049, + "logps/chosen": -320.04449462890625, + "logps/rejected": -315.7063903808594, + "loss": 0.5535, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.18441326916217804, + "rewards/margins": 0.4230337142944336, + "rewards/rejected": -0.6074470281600952, + "step": 4820 + }, + { + "epoch": 0.63, + "learning_rate": 1.7942242675276098e-06, + "logits/chosen": -2.54813814163208, + "logits/rejected": -2.5858755111694336, + "logps/chosen": -217.2032928466797, + "logps/rejected": -291.31768798828125, + "loss": 0.5967, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.23552434146404266, + "rewards/margins": 0.26469406485557556, + "rewards/rejected": -0.500218391418457, + "step": 4830 + }, + { + "epoch": 0.63, + "learning_rate": 1.783275575274856e-06, + "logits/chosen": -2.6607840061187744, + "logits/rejected": -2.5804603099823, + "logps/chosen": -345.90032958984375, + "logps/rejected": -292.3533630371094, + "loss": 0.5858, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.07496364414691925, + "rewards/margins": 0.43582382798194885, + "rewards/rejected": -0.5107874274253845, + "step": 4840 + }, + { + "epoch": 0.63, + "learning_rate": 1.7723418403200943e-06, + "logits/chosen": -2.649951457977295, + "logits/rejected": -2.559494733810425, + "logps/chosen": -356.70001220703125, + "logps/rejected": -309.3967590332031, + "loss": 0.5836, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.2421417236328125, + "rewards/margins": 0.306281715631485, + "rewards/rejected": -0.5484234094619751, + "step": 4850 + }, + { + "epoch": 0.64, + "learning_rate": 1.7614232908390748e-06, + "logits/chosen": -2.5764052867889404, + "logits/rejected": -2.4994382858276367, + "logps/chosen": -314.18359375, + "logps/rejected": -336.04644775390625, + "loss": 0.6323, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.25903838872909546, + "rewards/margins": 0.19894324243068695, + "rewards/rejected": -0.4579816460609436, + "step": 4860 + }, + { + "epoch": 0.64, + "learning_rate": 1.7505201546906398e-06, + "logits/chosen": -2.516021490097046, + "logits/rejected": -2.580906867980957, + "logps/chosen": -351.5046081542969, + "logps/rejected": -300.8806457519531, + "loss": 0.5847, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.26838454604148865, + "rewards/margins": 0.27423301339149475, + "rewards/rejected": -0.5426175594329834, + "step": 4870 + }, + { + "epoch": 0.64, + "learning_rate": 1.7396326594119717e-06, + "logits/chosen": -2.498110294342041, + "logits/rejected": -2.5909085273742676, + "logps/chosen": -282.8095703125, + "logps/rejected": -283.3281555175781, + "loss": 0.5658, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.13294625282287598, + "rewards/margins": 0.37030714750289917, + "rewards/rejected": -0.5032534003257751, + "step": 4880 + }, + { + "epoch": 0.64, + "learning_rate": 1.7287610322138449e-06, + "logits/chosen": -2.5078094005584717, + "logits/rejected": -2.500873565673828, + "logps/chosen": -319.1579284667969, + "logps/rejected": -317.5836486816406, + "loss": 0.5706, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.205757737159729, + "rewards/margins": 0.40956979990005493, + "rewards/rejected": -0.6153275370597839, + "step": 4890 + }, + { + "epoch": 0.64, + "learning_rate": 1.7179054999758817e-06, + "logits/chosen": -2.6467504501342773, + "logits/rejected": -2.6136515140533447, + "logps/chosen": -284.30902099609375, + "logps/rejected": -292.63519287109375, + "loss": 0.5841, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.28274065256118774, + "rewards/margins": 0.25528156757354736, + "rewards/rejected": -0.5380222797393799, + "step": 4900 + }, + { + "epoch": 0.64, + "learning_rate": 1.7070662892418225e-06, + "logits/chosen": -2.596738815307617, + "logits/rejected": -2.5798354148864746, + "logps/chosen": -261.60443115234375, + "logps/rejected": -283.7542419433594, + "loss": 0.5948, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.32567527890205383, + "rewards/margins": 0.2797202467918396, + "rewards/rejected": -0.6053955554962158, + "step": 4910 + }, + { + "epoch": 0.64, + "learning_rate": 1.6962436262147913e-06, + "logits/chosen": -2.4732182025909424, + "logits/rejected": -2.435441493988037, + "logps/chosen": -241.5220947265625, + "logps/rejected": -286.060302734375, + "loss": 0.6128, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.3028459846973419, + "rewards/margins": 0.2888830304145813, + "rewards/rejected": -0.5917289853096008, + "step": 4920 + }, + { + "epoch": 0.65, + "learning_rate": 1.6854377367525814e-06, + "logits/chosen": -2.4663939476013184, + "logits/rejected": -2.487776279449463, + "logps/chosen": -319.4354248046875, + "logps/rejected": -306.67352294921875, + "loss": 0.5711, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15108667314052582, + "rewards/margins": 0.3576280474662781, + "rewards/rejected": -0.5087146759033203, + "step": 4930 + }, + { + "epoch": 0.65, + "learning_rate": 1.6746488463629362e-06, + "logits/chosen": -2.7180588245391846, + "logits/rejected": -2.760690689086914, + "logps/chosen": -348.2672119140625, + "logps/rejected": -379.86920166015625, + "loss": 0.5902, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.26233556866645813, + "rewards/margins": 0.28122225403785706, + "rewards/rejected": -0.5435577630996704, + "step": 4940 + }, + { + "epoch": 0.65, + "learning_rate": 1.6638771801988483e-06, + "logits/chosen": -2.604320526123047, + "logits/rejected": -2.592014789581299, + "logps/chosen": -343.21807861328125, + "logps/rejected": -367.0710144042969, + "loss": 0.5935, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.20770473778247833, + "rewards/margins": 0.39076295495033264, + "rewards/rejected": -0.5984677076339722, + "step": 4950 + }, + { + "epoch": 0.65, + "learning_rate": 1.653122963053857e-06, + "logits/chosen": -2.466764450073242, + "logits/rejected": -2.38940167427063, + "logps/chosen": -309.2972106933594, + "logps/rejected": -305.632080078125, + "loss": 0.5599, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16606129705905914, + "rewards/margins": 0.3350328803062439, + "rewards/rejected": -0.5010942220687866, + "step": 4960 + }, + { + "epoch": 0.65, + "learning_rate": 1.6423864193573606e-06, + "logits/chosen": -2.334670305252075, + "logits/rejected": -2.3478381633758545, + "logps/chosen": -261.6419677734375, + "logps/rejected": -290.67047119140625, + "loss": 0.5637, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16364139318466187, + "rewards/margins": 0.3203355371952057, + "rewards/rejected": -0.48397690057754517, + "step": 4970 + }, + { + "epoch": 0.65, + "learning_rate": 1.6316677731699286e-06, + "logits/chosen": -2.607004165649414, + "logits/rejected": -2.4803566932678223, + "logps/chosen": -345.6315612792969, + "logps/rejected": -337.2192687988281, + "loss": 0.5759, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.35969680547714233, + "rewards/margins": 0.36782944202423096, + "rewards/rejected": -0.7275261878967285, + "step": 4980 + }, + { + "epoch": 0.65, + "learning_rate": 1.6209672481786302e-06, + "logits/chosen": -2.478020668029785, + "logits/rejected": -2.502074718475342, + "logps/chosen": -288.4017333984375, + "logps/rejected": -283.6263427734375, + "loss": 0.5328, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1178351417183876, + "rewards/margins": 0.4654744267463684, + "rewards/rejected": -0.583309531211853, + "step": 4990 + }, + { + "epoch": 0.65, + "learning_rate": 1.6102850676923616e-06, + "logits/chosen": -2.546351194381714, + "logits/rejected": -2.5766713619232178, + "logps/chosen": -293.3699645996094, + "logps/rejected": -316.5007629394531, + "loss": 0.6016, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2899512052536011, + "rewards/margins": 0.37485018372535706, + "rewards/rejected": -0.6648014187812805, + "step": 5000 + }, + { + "epoch": 0.66, + "learning_rate": 1.5996214546371888e-06, + "logits/chosen": -2.685042381286621, + "logits/rejected": -2.593893051147461, + "logps/chosen": -272.0263366699219, + "logps/rejected": -256.42279052734375, + "loss": 0.5752, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16017282009124756, + "rewards/margins": 0.3072182834148407, + "rewards/rejected": -0.46739110350608826, + "step": 5010 + }, + { + "epoch": 0.66, + "learning_rate": 1.588976631551697e-06, + "logits/chosen": -2.617985248565674, + "logits/rejected": -2.576514482498169, + "logps/chosen": -304.63189697265625, + "logps/rejected": -308.66558837890625, + "loss": 0.5844, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.190621480345726, + "rewards/margins": 0.27106973528862, + "rewards/rejected": -0.4616912007331848, + "step": 5020 + }, + { + "epoch": 0.66, + "learning_rate": 1.5783508205823412e-06, + "logits/chosen": -2.4902517795562744, + "logits/rejected": -2.58402156829834, + "logps/chosen": -277.86285400390625, + "logps/rejected": -292.60687255859375, + "loss": 0.6241, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16955803334712982, + "rewards/margins": 0.3012115955352783, + "rewards/rejected": -0.47076964378356934, + "step": 5030 + }, + { + "epoch": 0.66, + "learning_rate": 1.5677442434788143e-06, + "logits/chosen": -2.6300415992736816, + "logits/rejected": -2.523313045501709, + "logps/chosen": -319.5474548339844, + "logps/rejected": -330.72418212890625, + "loss": 0.5937, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16097375750541687, + "rewards/margins": 0.34245041012763977, + "rewards/rejected": -0.5034242272377014, + "step": 5040 + }, + { + "epoch": 0.66, + "learning_rate": 1.5571571215894181e-06, + "logits/chosen": -2.7039828300476074, + "logits/rejected": -2.668703556060791, + "logps/chosen": -307.61248779296875, + "logps/rejected": -362.8529052734375, + "loss": 0.5754, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.25565141439437866, + "rewards/margins": 0.298455148935318, + "rewards/rejected": -0.554106593132019, + "step": 5050 + }, + { + "epoch": 0.66, + "learning_rate": 1.5465896758564452e-06, + "logits/chosen": -2.6605722904205322, + "logits/rejected": -2.6176297664642334, + "logps/chosen": -319.3802795410156, + "logps/rejected": -349.5055236816406, + "loss": 0.6221, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1468852460384369, + "rewards/margins": 0.27727001905441284, + "rewards/rejected": -0.4241552948951721, + "step": 5060 + }, + { + "epoch": 0.66, + "learning_rate": 1.5360421268115653e-06, + "logits/chosen": -2.662797212600708, + "logits/rejected": -2.5455074310302734, + "logps/chosen": -338.2200622558594, + "logps/rejected": -312.7371826171875, + "loss": 0.5569, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.25399890542030334, + "rewards/margins": 0.3884117305278778, + "rewards/rejected": -0.6424106359481812, + "step": 5070 + }, + { + "epoch": 0.66, + "learning_rate": 1.5255146945712267e-06, + "logits/chosen": -2.5705513954162598, + "logits/rejected": -2.4772160053253174, + "logps/chosen": -326.19989013671875, + "logps/rejected": -275.66412353515625, + "loss": 0.5953, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21252810955047607, + "rewards/margins": 0.3057176470756531, + "rewards/rejected": -0.5182457566261292, + "step": 5080 + }, + { + "epoch": 0.67, + "learning_rate": 1.5150075988320594e-06, + "logits/chosen": -2.485109806060791, + "logits/rejected": -2.40926194190979, + "logps/chosen": -261.7632141113281, + "logps/rejected": -284.9822692871094, + "loss": 0.5824, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.23927578330039978, + "rewards/margins": 0.3335501253604889, + "rewards/rejected": -0.5728258490562439, + "step": 5090 + }, + { + "epoch": 0.67, + "learning_rate": 1.5045210588662929e-06, + "logits/chosen": -2.541929244995117, + "logits/rejected": -2.529249668121338, + "logps/chosen": -272.0650634765625, + "logps/rejected": -292.97467041015625, + "loss": 0.5757, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11239515244960785, + "rewards/margins": 0.4191061854362488, + "rewards/rejected": -0.5315013527870178, + "step": 5100 + }, + { + "epoch": 0.67, + "learning_rate": 1.4940552935171781e-06, + "logits/chosen": -2.7147622108459473, + "logits/rejected": -2.5909676551818848, + "logps/chosen": -323.33221435546875, + "logps/rejected": -350.9828796386719, + "loss": 0.5739, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.25958338379859924, + "rewards/margins": 0.29021376371383667, + "rewards/rejected": -0.5497971773147583, + "step": 5110 + }, + { + "epoch": 0.67, + "learning_rate": 1.483610521194419e-06, + "logits/chosen": -2.485908269882202, + "logits/rejected": -2.4038195610046387, + "logps/chosen": -340.5307312011719, + "logps/rejected": -317.41571044921875, + "loss": 0.5661, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1522163301706314, + "rewards/margins": 0.5749739408493042, + "rewards/rejected": -0.7271903157234192, + "step": 5120 + }, + { + "epoch": 0.67, + "learning_rate": 1.4731869598696226e-06, + "logits/chosen": -2.465968608856201, + "logits/rejected": -2.5231192111968994, + "logps/chosen": -259.185546875, + "logps/rejected": -318.861572265625, + "loss": 0.5764, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.21996121108531952, + "rewards/margins": 0.2890729308128357, + "rewards/rejected": -0.5090342164039612, + "step": 5130 + }, + { + "epoch": 0.67, + "learning_rate": 1.4627848270717387e-06, + "logits/chosen": -2.5720362663269043, + "logits/rejected": -2.435134172439575, + "logps/chosen": -305.77734375, + "logps/rejected": -305.54583740234375, + "loss": 0.5563, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.2069060057401657, + "rewards/margins": 0.3971417546272278, + "rewards/rejected": -0.6040477752685547, + "step": 5140 + }, + { + "epoch": 0.67, + "learning_rate": 1.4524043398825277e-06, + "logits/chosen": -2.3851141929626465, + "logits/rejected": -2.382739305496216, + "logps/chosen": -343.54998779296875, + "logps/rejected": -292.6429138183594, + "loss": 0.6035, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.19103741645812988, + "rewards/margins": 0.27970466017723083, + "rewards/rejected": -0.4707420766353607, + "step": 5150 + }, + { + "epoch": 0.68, + "learning_rate": 1.4420457149320299e-06, + "logits/chosen": -2.481121778488159, + "logits/rejected": -2.4870851039886475, + "logps/chosen": -281.44415283203125, + "logps/rejected": -332.45391845703125, + "loss": 0.6341, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.2218354046344757, + "rewards/margins": 0.15597988665103912, + "rewards/rejected": -0.377815306186676, + "step": 5160 + }, + { + "epoch": 0.68, + "learning_rate": 1.431709168394042e-06, + "logits/chosen": -2.6676456928253174, + "logits/rejected": -2.6511683464050293, + "logps/chosen": -373.96893310546875, + "logps/rejected": -315.47015380859375, + "loss": 0.5551, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3234941363334656, + "rewards/margins": 0.19331450760364532, + "rewards/rejected": -0.5168086290359497, + "step": 5170 + }, + { + "epoch": 0.68, + "learning_rate": 1.4213949159816059e-06, + "logits/chosen": -2.498394727706909, + "logits/rejected": -2.3926968574523926, + "logps/chosen": -276.31793212890625, + "logps/rejected": -293.73260498046875, + "loss": 0.5533, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19067999720573425, + "rewards/margins": 0.4055241048336029, + "rewards/rejected": -0.5962041616439819, + "step": 5180 + }, + { + "epoch": 0.68, + "learning_rate": 1.4111031729425103e-06, + "logits/chosen": -2.5006566047668457, + "logits/rejected": -2.4794576168060303, + "logps/chosen": -324.97137451171875, + "logps/rejected": -303.35662841796875, + "loss": 0.5635, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.24338562786579132, + "rewards/margins": 0.33955326676368713, + "rewards/rejected": -0.5829388499259949, + "step": 5190 + }, + { + "epoch": 0.68, + "learning_rate": 1.4008341540547965e-06, + "logits/chosen": -2.4405434131622314, + "logits/rejected": -2.5150179862976074, + "logps/chosen": -291.26898193359375, + "logps/rejected": -311.96185302734375, + "loss": 0.5954, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16561874747276306, + "rewards/margins": 0.2473074197769165, + "rewards/rejected": -0.41292619705200195, + "step": 5200 + }, + { + "epoch": 0.68, + "learning_rate": 1.3905880736222737e-06, + "logits/chosen": -2.4981284141540527, + "logits/rejected": -2.52009916305542, + "logps/chosen": -255.4973602294922, + "logps/rejected": -305.76678466796875, + "loss": 0.5787, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.20114591717720032, + "rewards/margins": 0.2925758957862854, + "rewards/rejected": -0.4937218725681305, + "step": 5210 + }, + { + "epoch": 0.68, + "learning_rate": 1.3803651454700531e-06, + "logits/chosen": -2.5396883487701416, + "logits/rejected": -2.4904818534851074, + "logps/chosen": -298.5966491699219, + "logps/rejected": -282.314208984375, + "loss": 0.5623, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.19929958879947662, + "rewards/margins": 0.3365853428840637, + "rewards/rejected": -0.5358849763870239, + "step": 5220 + }, + { + "epoch": 0.68, + "learning_rate": 1.3701655829400773e-06, + "logits/chosen": -2.695084571838379, + "logits/rejected": -2.5825228691101074, + "logps/chosen": -399.9344787597656, + "logps/rejected": -328.28179931640625, + "loss": 0.5791, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.23696422576904297, + "rewards/margins": 0.32420456409454346, + "rewards/rejected": -0.5611687898635864, + "step": 5230 + }, + { + "epoch": 0.69, + "learning_rate": 1.3599895988866756e-06, + "logits/chosen": -2.622859477996826, + "logits/rejected": -2.580024242401123, + "logps/chosen": -272.3352355957031, + "logps/rejected": -299.87139892578125, + "loss": 0.6425, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.22998254001140594, + "rewards/margins": 0.18282505869865417, + "rewards/rejected": -0.4128076434135437, + "step": 5240 + }, + { + "epoch": 0.69, + "learning_rate": 1.3498374056721198e-06, + "logits/chosen": -2.707981824874878, + "logits/rejected": -2.7110109329223633, + "logps/chosen": -274.9527587890625, + "logps/rejected": -382.5824279785156, + "loss": 0.5442, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1651863157749176, + "rewards/margins": 0.3817494511604309, + "rewards/rejected": -0.5469357371330261, + "step": 5250 + }, + { + "epoch": 0.69, + "learning_rate": 1.3397092151621883e-06, + "logits/chosen": -2.554539442062378, + "logits/rejected": -2.482882022857666, + "logps/chosen": -286.2304992675781, + "logps/rejected": -266.2547607421875, + "loss": 0.6205, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.26898160576820374, + "rewards/margins": 0.21374158561229706, + "rewards/rejected": -0.4827231764793396, + "step": 5260 + }, + { + "epoch": 0.69, + "learning_rate": 1.3296052387217484e-06, + "logits/chosen": -2.45316481590271, + "logits/rejected": -2.5683462619781494, + "logps/chosen": -238.8728790283203, + "logps/rejected": -235.2001495361328, + "loss": 0.5939, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.16806179285049438, + "rewards/margins": 0.333972692489624, + "rewards/rejected": -0.5020344853401184, + "step": 5270 + }, + { + "epoch": 0.69, + "learning_rate": 1.3195256872103476e-06, + "logits/chosen": -2.648393392562866, + "logits/rejected": -2.59840726852417, + "logps/chosen": -324.327880859375, + "logps/rejected": -378.9349670410156, + "loss": 0.6047, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.1936226338148117, + "rewards/margins": 0.20749232172966003, + "rewards/rejected": -0.40111494064331055, + "step": 5280 + }, + { + "epoch": 0.69, + "learning_rate": 1.3094707709778068e-06, + "logits/chosen": -2.4787323474884033, + "logits/rejected": -2.448561906814575, + "logps/chosen": -247.78439331054688, + "logps/rejected": -260.88128662109375, + "loss": 0.5579, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1471039354801178, + "rewards/margins": 0.37695735692977905, + "rewards/rejected": -0.5240613222122192, + "step": 5290 + }, + { + "epoch": 0.69, + "learning_rate": 1.2994406998598364e-06, + "logits/chosen": -2.550649642944336, + "logits/rejected": -2.446993350982666, + "logps/chosen": -208.6945343017578, + "logps/rejected": -247.4278564453125, + "loss": 0.5701, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17239803075790405, + "rewards/margins": 0.4012460708618164, + "rewards/rejected": -0.5736441016197205, + "step": 5300 + }, + { + "epoch": 0.69, + "learning_rate": 1.2894356831736558e-06, + "logits/chosen": -2.5386788845062256, + "logits/rejected": -2.5003950595855713, + "logps/chosen": -270.95428466796875, + "logps/rejected": -281.563720703125, + "loss": 0.5692, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.2243034839630127, + "rewards/margins": 0.36400306224823, + "rewards/rejected": -0.5883064866065979, + "step": 5310 + }, + { + "epoch": 0.7, + "learning_rate": 1.2794559297136203e-06, + "logits/chosen": -2.4010531902313232, + "logits/rejected": -2.4677743911743164, + "logps/chosen": -268.36102294921875, + "logps/rejected": -312.61871337890625, + "loss": 0.6012, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.11416293680667877, + "rewards/margins": 0.3250843286514282, + "rewards/rejected": -0.4392472803592682, + "step": 5320 + }, + { + "epoch": 0.7, + "learning_rate": 1.2695016477468724e-06, + "logits/chosen": -2.6123104095458984, + "logits/rejected": -2.658905506134033, + "logps/chosen": -315.5183410644531, + "logps/rejected": -280.52191162109375, + "loss": 0.645, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1765400469303131, + "rewards/margins": 0.17122364044189453, + "rewards/rejected": -0.34776371717453003, + "step": 5330 + }, + { + "epoch": 0.7, + "learning_rate": 1.2595730450089874e-06, + "logits/chosen": -2.68192720413208, + "logits/rejected": -2.6289165019989014, + "logps/chosen": -316.01898193359375, + "logps/rejected": -322.196533203125, + "loss": 0.5852, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16447719931602478, + "rewards/margins": 0.36463385820388794, + "rewards/rejected": -0.5291110873222351, + "step": 5340 + }, + { + "epoch": 0.7, + "learning_rate": 1.2496703286996433e-06, + "logits/chosen": -2.685610055923462, + "logits/rejected": -2.589296340942383, + "logps/chosen": -365.1530456542969, + "logps/rejected": -335.67156982421875, + "loss": 0.6027, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.22830632328987122, + "rewards/margins": 0.3892638087272644, + "rewards/rejected": -0.6175701022148132, + "step": 5350 + }, + { + "epoch": 0.7, + "learning_rate": 1.2397937054782961e-06, + "logits/chosen": -2.555673122406006, + "logits/rejected": -2.625791549682617, + "logps/chosen": -346.9773864746094, + "logps/rejected": -334.68756103515625, + "loss": 0.5705, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.23506096005439758, + "rewards/margins": 0.3681856393814087, + "rewards/rejected": -0.6032465696334839, + "step": 5360 + }, + { + "epoch": 0.7, + "learning_rate": 1.2299433814598635e-06, + "logits/chosen": -2.5669150352478027, + "logits/rejected": -2.6253628730773926, + "logps/chosen": -268.91680908203125, + "logps/rejected": -296.8352355957031, + "loss": 0.5799, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.10326936095952988, + "rewards/margins": 0.25865739583969116, + "rewards/rejected": -0.36192673444747925, + "step": 5370 + }, + { + "epoch": 0.7, + "learning_rate": 1.2201195622104265e-06, + "logits/chosen": -2.5529732704162598, + "logits/rejected": -2.570631504058838, + "logps/chosen": -280.7418212890625, + "logps/rejected": -277.921630859375, + "loss": 0.5804, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.21117191016674042, + "rewards/margins": 0.3797195553779602, + "rewards/rejected": -0.5908914804458618, + "step": 5380 + }, + { + "epoch": 0.71, + "learning_rate": 1.2103224527429417e-06, + "logits/chosen": -2.5260720252990723, + "logits/rejected": -2.5511975288391113, + "logps/chosen": -303.44635009765625, + "logps/rejected": -246.2676239013672, + "loss": 0.6542, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.12673085927963257, + "rewards/margins": 0.29531019926071167, + "rewards/rejected": -0.42204099893569946, + "step": 5390 + }, + { + "epoch": 0.71, + "learning_rate": 1.2005522575129559e-06, + "logits/chosen": -2.6706199645996094, + "logits/rejected": -2.5881125926971436, + "logps/chosen": -391.4795837402344, + "logps/rejected": -295.01031494140625, + "loss": 0.655, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2009950876235962, + "rewards/margins": 0.23872563242912292, + "rewards/rejected": -0.4397208094596863, + "step": 5400 + }, + { + "epoch": 0.71, + "learning_rate": 1.1908091804143469e-06, + "logits/chosen": -2.438290596008301, + "logits/rejected": -2.404784679412842, + "logps/chosen": -300.5682373046875, + "logps/rejected": -316.4365234375, + "loss": 0.6282, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2771514058113098, + "rewards/margins": 0.25950735807418823, + "rewards/rejected": -0.536658763885498, + "step": 5410 + }, + { + "epoch": 0.71, + "learning_rate": 1.1810934247750649e-06, + "logits/chosen": -2.5039753913879395, + "logits/rejected": -2.472374200820923, + "logps/chosen": -355.4488220214844, + "logps/rejected": -306.5274353027344, + "loss": 0.5836, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.11722154915332794, + "rewards/margins": 0.4215324819087982, + "rewards/rejected": -0.5387539863586426, + "step": 5420 + }, + { + "epoch": 0.71, + "learning_rate": 1.1714051933528881e-06, + "logits/chosen": -2.6620097160339355, + "logits/rejected": -2.5606446266174316, + "logps/chosen": -340.6926574707031, + "logps/rejected": -322.7391052246094, + "loss": 0.5843, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.18550732731819153, + "rewards/margins": 0.3209478259086609, + "rewards/rejected": -0.50645512342453, + "step": 5430 + }, + { + "epoch": 0.71, + "learning_rate": 1.161744688331192e-06, + "logits/chosen": -2.569016695022583, + "logits/rejected": -2.427980422973633, + "logps/chosen": -297.58782958984375, + "logps/rejected": -343.2862854003906, + "loss": 0.622, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.30308234691619873, + "rewards/margins": 0.1463664025068283, + "rewards/rejected": -0.44944876432418823, + "step": 5440 + }, + { + "epoch": 0.71, + "learning_rate": 1.152112111314733e-06, + "logits/chosen": -2.584120512008667, + "logits/rejected": -2.5221121311187744, + "logps/chosen": -292.50823974609375, + "logps/rejected": -288.62921142578125, + "loss": 0.6346, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.20627093315124512, + "rewards/margins": 0.20762047171592712, + "rewards/rejected": -0.413891464471817, + "step": 5450 + }, + { + "epoch": 0.71, + "learning_rate": 1.142507663325439e-06, + "logits/chosen": -2.5839767456054688, + "logits/rejected": -2.5707592964172363, + "logps/chosen": -337.36578369140625, + "logps/rejected": -340.32684326171875, + "loss": 0.5433, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1801554262638092, + "rewards/margins": 0.41654491424560547, + "rewards/rejected": -0.5967002511024475, + "step": 5460 + }, + { + "epoch": 0.72, + "learning_rate": 1.132931544798211e-06, + "logits/chosen": -2.521233558654785, + "logits/rejected": -2.5143444538116455, + "logps/chosen": -303.11968994140625, + "logps/rejected": -321.09661865234375, + "loss": 0.5342, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.12275087833404541, + "rewards/margins": 0.49955272674560547, + "rewards/rejected": -0.6223036050796509, + "step": 5470 + }, + { + "epoch": 0.72, + "learning_rate": 1.1233839555767482e-06, + "logits/chosen": -2.4385015964508057, + "logits/rejected": -2.3833563327789307, + "logps/chosen": -290.13604736328125, + "logps/rejected": -249.4829559326172, + "loss": 0.5508, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15207554399967194, + "rewards/margins": 0.4083792567253113, + "rewards/rejected": -0.5604548454284668, + "step": 5480 + }, + { + "epoch": 0.72, + "learning_rate": 1.1138650949093668e-06, + "logits/chosen": -2.5767691135406494, + "logits/rejected": -2.4980454444885254, + "logps/chosen": -255.419189453125, + "logps/rejected": -224.4200439453125, + "loss": 0.6602, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.31322962045669556, + "rewards/margins": 0.19506050646305084, + "rewards/rejected": -0.5082900524139404, + "step": 5490 + }, + { + "epoch": 0.72, + "learning_rate": 1.1043751614448543e-06, + "logits/chosen": -2.431666135787964, + "logits/rejected": -2.3092381954193115, + "logps/chosen": -237.71817016601562, + "logps/rejected": -221.69140625, + "loss": 0.5965, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1798931211233139, + "rewards/margins": 0.2392689287662506, + "rewards/rejected": -0.4191620349884033, + "step": 5500 + }, + { + "epoch": 0.72, + "learning_rate": 1.0949143532283107e-06, + "logits/chosen": -2.580761432647705, + "logits/rejected": -2.549431800842285, + "logps/chosen": -321.2472839355469, + "logps/rejected": -338.0050048828125, + "loss": 0.5863, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.2106788456439972, + "rewards/margins": 0.34467029571533203, + "rewards/rejected": -0.5553491711616516, + "step": 5510 + }, + { + "epoch": 0.72, + "learning_rate": 1.0854828676970275e-06, + "logits/chosen": -2.4911673069000244, + "logits/rejected": -2.5411815643310547, + "logps/chosen": -248.7816925048828, + "logps/rejected": -274.0561218261719, + "loss": 0.5994, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.13380002975463867, + "rewards/margins": 0.3159855604171753, + "rewards/rejected": -0.4497855305671692, + "step": 5520 + }, + { + "epoch": 0.72, + "learning_rate": 1.076080901676361e-06, + "logits/chosen": -2.6060454845428467, + "logits/rejected": -2.626995325088501, + "logps/chosen": -280.73358154296875, + "logps/rejected": -289.40032958984375, + "loss": 0.6006, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2465798407793045, + "rewards/margins": 0.2739890515804291, + "rewards/rejected": -0.5205689072608948, + "step": 5530 + }, + { + "epoch": 0.72, + "learning_rate": 1.0667086513756234e-06, + "logits/chosen": -2.6200010776519775, + "logits/rejected": -2.5084142684936523, + "logps/chosen": -293.21099853515625, + "logps/rejected": -301.6272888183594, + "loss": 0.5946, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.18248774111270905, + "rewards/margins": 0.34508463740348816, + "rewards/rejected": -0.5275723338127136, + "step": 5540 + }, + { + "epoch": 0.73, + "learning_rate": 1.0573663123839912e-06, + "logits/chosen": -2.5958545207977295, + "logits/rejected": -2.530036449432373, + "logps/chosen": -302.2039489746094, + "logps/rejected": -267.8409118652344, + "loss": 0.5642, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1686379760503769, + "rewards/margins": 0.46634167432785034, + "rewards/rejected": -0.6349795460700989, + "step": 5550 + }, + { + "epoch": 0.73, + "learning_rate": 1.0480540796664251e-06, + "logits/chosen": -2.4992308616638184, + "logits/rejected": -2.491286039352417, + "logps/chosen": -319.95916748046875, + "logps/rejected": -346.5733337402344, + "loss": 0.5925, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.28914397954940796, + "rewards/margins": 0.20945701003074646, + "rewards/rejected": -0.4986010193824768, + "step": 5560 + }, + { + "epoch": 0.73, + "learning_rate": 1.0387721475595978e-06, + "logits/chosen": -2.5275931358337402, + "logits/rejected": -2.4925732612609863, + "logps/chosen": -221.79342651367188, + "logps/rejected": -234.3103485107422, + "loss": 0.6016, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.24439530074596405, + "rewards/margins": 0.23605410754680634, + "rewards/rejected": -0.480449378490448, + "step": 5570 + }, + { + "epoch": 0.73, + "learning_rate": 1.0295207097678378e-06, + "logits/chosen": -2.433387279510498, + "logits/rejected": -2.4333786964416504, + "logps/chosen": -226.1207275390625, + "logps/rejected": -324.7813415527344, + "loss": 0.5561, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.16782227158546448, + "rewards/margins": 0.3883552551269531, + "rewards/rejected": -0.5561774969100952, + "step": 5580 + }, + { + "epoch": 0.73, + "learning_rate": 1.0202999593590924e-06, + "logits/chosen": -2.5502381324768066, + "logits/rejected": -2.5282599925994873, + "logps/chosen": -267.7656555175781, + "logps/rejected": -308.3643493652344, + "loss": 0.5295, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.12993329763412476, + "rewards/margins": 0.4945352077484131, + "rewards/rejected": -0.6244684457778931, + "step": 5590 + }, + { + "epoch": 0.73, + "learning_rate": 1.011110088760891e-06, + "logits/chosen": -2.455394744873047, + "logits/rejected": -2.5353024005889893, + "logps/chosen": -244.3286590576172, + "logps/rejected": -283.96429443359375, + "loss": 0.5477, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.20858506858348846, + "rewards/margins": 0.46013203263282776, + "rewards/rejected": -0.6687171459197998, + "step": 5600 + }, + { + "epoch": 0.73, + "learning_rate": 1.0019512897563347e-06, + "logits/chosen": -2.6215782165527344, + "logits/rejected": -2.4959828853607178, + "logps/chosen": -300.98876953125, + "logps/rejected": -297.37158203125, + "loss": 0.6051, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14473457634449005, + "rewards/margins": 0.34757936000823975, + "rewards/rejected": -0.492313951253891, + "step": 5610 + }, + { + "epoch": 0.74, + "learning_rate": 9.928237534800935e-07, + "logits/chosen": -2.6514501571655273, + "logits/rejected": -2.6429600715637207, + "logps/chosen": -281.0267028808594, + "logps/rejected": -294.16644287109375, + "loss": 0.5812, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.11892740428447723, + "rewards/margins": 0.35566264390945435, + "rewards/rejected": -0.4745900630950928, + "step": 5620 + }, + { + "epoch": 0.74, + "learning_rate": 9.837276704144174e-07, + "logits/chosen": -2.627713918685913, + "logits/rejected": -2.588146686553955, + "logps/chosen": -265.66455078125, + "logps/rejected": -259.3338928222656, + "loss": 0.5904, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1974610537290573, + "rewards/margins": 0.2754455506801605, + "rewards/rejected": -0.47290658950805664, + "step": 5630 + }, + { + "epoch": 0.74, + "learning_rate": 9.746632303851569e-07, + "logits/chosen": -2.6073737144470215, + "logits/rejected": -2.5223560333251953, + "logps/chosen": -310.0323791503906, + "logps/rejected": -273.57391357421875, + "loss": 0.6077, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2162187099456787, + "rewards/margins": 0.34184861183166504, + "rewards/rejected": -0.5580673217773438, + "step": 5640 + }, + { + "epoch": 0.74, + "learning_rate": 9.65630622557809e-07, + "logits/chosen": -2.5930376052856445, + "logits/rejected": -2.4731199741363525, + "logps/chosen": -358.55145263671875, + "logps/rejected": -344.0464172363281, + "loss": 0.6056, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2608993649482727, + "rewards/margins": 0.24014616012573242, + "rewards/rejected": -0.5010455846786499, + "step": 5650 + }, + { + "epoch": 0.74, + "learning_rate": 9.56630035433561e-07, + "logits/chosen": -2.6305251121520996, + "logits/rejected": -2.5761353969573975, + "logps/chosen": -330.01239013671875, + "logps/rejected": -283.33489990234375, + "loss": 0.5968, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1723465621471405, + "rewards/margins": 0.23930080235004425, + "rewards/rejected": -0.41164731979370117, + "step": 5660 + }, + { + "epoch": 0.74, + "learning_rate": 9.476616568453659e-07, + "logits/chosen": -2.5701346397399902, + "logits/rejected": -2.505082368850708, + "logps/chosen": -317.55303955078125, + "logps/rejected": -285.5252990722656, + "loss": 0.537, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2099696844816208, + "rewards/margins": 0.4133245348930359, + "rewards/rejected": -0.6232942342758179, + "step": 5670 + }, + { + "epoch": 0.74, + "learning_rate": 9.387256739540162e-07, + "logits/chosen": -2.589440107345581, + "logits/rejected": -2.581523895263672, + "logps/chosen": -336.957763671875, + "logps/rejected": -325.838623046875, + "loss": 0.5688, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15233464539051056, + "rewards/margins": 0.35094717144966125, + "rewards/rejected": -0.5032817721366882, + "step": 5680 + }, + { + "epoch": 0.74, + "learning_rate": 9.298222732442377e-07, + "logits/chosen": -2.658050537109375, + "logits/rejected": -2.5099217891693115, + "logps/chosen": -319.69879150390625, + "logps/rejected": -303.26031494140625, + "loss": 0.5772, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.15199032425880432, + "rewards/margins": 0.4483245313167572, + "rewards/rejected": -0.600314736366272, + "step": 5690 + }, + { + "epoch": 0.75, + "learning_rate": 9.20951640520803e-07, + "logits/chosen": -2.556957244873047, + "logits/rejected": -2.5528512001037598, + "logps/chosen": -270.5416259765625, + "logps/rejected": -284.24444580078125, + "loss": 0.578, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.15435631573200226, + "rewards/margins": 0.3883664011955261, + "rewards/rejected": -0.542722761631012, + "step": 5700 + }, + { + "epoch": 0.75, + "learning_rate": 9.121139609046484e-07, + "logits/chosen": -2.516080379486084, + "logits/rejected": -2.489114999771118, + "logps/chosen": -303.9247741699219, + "logps/rejected": -362.7187805175781, + "loss": 0.6415, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21308767795562744, + "rewards/margins": 0.3083495497703552, + "rewards/rejected": -0.5214372873306274, + "step": 5710 + }, + { + "epoch": 0.75, + "learning_rate": 9.033094188290121e-07, + "logits/chosen": -2.392362356185913, + "logits/rejected": -2.425485134124756, + "logps/chosen": -325.4200439453125, + "logps/rejected": -310.832275390625, + "loss": 0.6042, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2987435460090637, + "rewards/margins": 0.34238141775131226, + "rewards/rejected": -0.641124963760376, + "step": 5720 + }, + { + "epoch": 0.75, + "learning_rate": 8.945381980355889e-07, + "logits/chosen": -2.576991558074951, + "logits/rejected": -2.5154693126678467, + "logps/chosen": -310.90142822265625, + "logps/rejected": -355.0868225097656, + "loss": 0.6146, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.24633660912513733, + "rewards/margins": 0.30161046981811523, + "rewards/rejected": -0.5479470491409302, + "step": 5730 + }, + { + "epoch": 0.75, + "learning_rate": 8.858004815706919e-07, + "logits/chosen": -2.481250762939453, + "logits/rejected": -2.510680675506592, + "logps/chosen": -277.41778564453125, + "logps/rejected": -293.6460266113281, + "loss": 0.5435, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.20409002900123596, + "rewards/margins": 0.4168508052825928, + "rewards/rejected": -0.6209408044815063, + "step": 5740 + }, + { + "epoch": 0.75, + "learning_rate": 8.77096451781432e-07, + "logits/chosen": -2.537360668182373, + "logits/rejected": -2.456080913543701, + "logps/chosen": -254.56448364257812, + "logps/rejected": -274.6441345214844, + "loss": 0.6138, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2944655120372772, + "rewards/margins": 0.24884769320487976, + "rewards/rejected": -0.543313205242157, + "step": 5750 + }, + { + "epoch": 0.75, + "learning_rate": 8.684262903119165e-07, + "logits/chosen": -2.5056889057159424, + "logits/rejected": -2.6055495738983154, + "logps/chosen": -276.7388610839844, + "logps/rejected": -314.30108642578125, + "loss": 0.6008, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.23513415455818176, + "rewards/margins": 0.2173004448413849, + "rewards/rejected": -0.45243462920188904, + "step": 5760 + }, + { + "epoch": 0.76, + "learning_rate": 8.597901780994525e-07, + "logits/chosen": -2.653364658355713, + "logits/rejected": -2.60335111618042, + "logps/chosen": -279.13238525390625, + "logps/rejected": -248.5230255126953, + "loss": 0.626, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2263440191745758, + "rewards/margins": 0.20855498313903809, + "rewards/rejected": -0.4348990321159363, + "step": 5770 + }, + { + "epoch": 0.76, + "learning_rate": 8.511882953707773e-07, + "logits/chosen": -2.5802361965179443, + "logits/rejected": -2.5868887901306152, + "logps/chosen": -342.547119140625, + "logps/rejected": -336.5263671875, + "loss": 0.596, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.21341674029827118, + "rewards/margins": 0.34919142723083496, + "rewards/rejected": -0.5626082420349121, + "step": 5780 + }, + { + "epoch": 0.76, + "learning_rate": 8.426208216382944e-07, + "logits/chosen": -2.4884839057922363, + "logits/rejected": -2.538118362426758, + "logps/chosen": -254.6638946533203, + "logps/rejected": -303.5052490234375, + "loss": 0.5528, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.20429837703704834, + "rewards/margins": 0.37576302886009216, + "rewards/rejected": -0.5800613760948181, + "step": 5790 + }, + { + "epoch": 0.76, + "learning_rate": 8.340879356963245e-07, + "logits/chosen": -2.4506642818450928, + "logits/rejected": -2.382664203643799, + "logps/chosen": -295.90997314453125, + "logps/rejected": -309.56756591796875, + "loss": 0.5646, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.23028168082237244, + "rewards/margins": 0.252223402261734, + "rewards/rejected": -0.48250502347946167, + "step": 5800 + }, + { + "epoch": 0.76, + "learning_rate": 8.255898156173777e-07, + "logits/chosen": -2.572566270828247, + "logits/rejected": -2.5592901706695557, + "logps/chosen": -250.8806610107422, + "logps/rejected": -271.90069580078125, + "loss": 0.5678, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.18372826278209686, + "rewards/margins": 0.3871956765651703, + "rewards/rejected": -0.570923924446106, + "step": 5810 + }, + { + "epoch": 0.76, + "learning_rate": 8.171266387484389e-07, + "logits/chosen": -2.6025376319885254, + "logits/rejected": -2.593602418899536, + "logps/chosen": -296.9878845214844, + "logps/rejected": -274.2995300292969, + "loss": 0.6177, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.19963833689689636, + "rewards/margins": 0.2611692547798157, + "rewards/rejected": -0.46080756187438965, + "step": 5820 + }, + { + "epoch": 0.76, + "learning_rate": 8.086985817072604e-07, + "logits/chosen": -2.5865213871002197, + "logits/rejected": -2.521101713180542, + "logps/chosen": -290.48248291015625, + "logps/rejected": -290.9474182128906, + "loss": 0.5939, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.16447794437408447, + "rewards/margins": 0.32579711079597473, + "rewards/rejected": -0.4902750849723816, + "step": 5830 + }, + { + "epoch": 0.76, + "learning_rate": 8.003058203786835e-07, + "logits/chosen": -2.486022472381592, + "logits/rejected": -2.4313018321990967, + "logps/chosen": -333.7433776855469, + "logps/rejected": -302.3613586425781, + "loss": 0.622, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.27535480260849, + "rewards/margins": 0.2524801790714264, + "rewards/rejected": -0.527834951877594, + "step": 5840 + }, + { + "epoch": 0.77, + "learning_rate": 7.91948529910963e-07, + "logits/chosen": -2.485321521759033, + "logits/rejected": -2.4735782146453857, + "logps/chosen": -293.6622009277344, + "logps/rejected": -316.9804992675781, + "loss": 0.5743, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.19359636306762695, + "rewards/margins": 0.4723263382911682, + "rewards/rejected": -0.6659227013587952, + "step": 5850 + }, + { + "epoch": 0.77, + "learning_rate": 7.836268847121126e-07, + "logits/chosen": -2.5226197242736816, + "logits/rejected": -2.5177345275878906, + "logps/chosen": -333.4381103515625, + "logps/rejected": -344.66888427734375, + "loss": 0.6019, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2280873954296112, + "rewards/margins": 0.2676352858543396, + "rewards/rejected": -0.4957226812839508, + "step": 5860 + }, + { + "epoch": 0.77, + "learning_rate": 7.753410584462681e-07, + "logits/chosen": -2.4178872108459473, + "logits/rejected": -2.3296265602111816, + "logps/chosen": -314.51025390625, + "logps/rejected": -292.57550048828125, + "loss": 0.5726, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -0.19157357513904572, + "rewards/margins": 0.46646785736083984, + "rewards/rejected": -0.6580413579940796, + "step": 5870 + }, + { + "epoch": 0.77, + "learning_rate": 7.670912240300596e-07, + "logits/chosen": -2.529723644256592, + "logits/rejected": -2.554422616958618, + "logps/chosen": -318.37469482421875, + "logps/rejected": -314.1541442871094, + "loss": 0.5606, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.09865571558475494, + "rewards/margins": 0.3737652599811554, + "rewards/rejected": -0.47242099046707153, + "step": 5880 + }, + { + "epoch": 0.77, + "learning_rate": 7.588775536290035e-07, + "logits/chosen": -2.5802218914031982, + "logits/rejected": -2.4884378910064697, + "logps/chosen": -291.59027099609375, + "logps/rejected": -292.0467529296875, + "loss": 0.5638, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.22754530608654022, + "rewards/margins": 0.4772590100765228, + "rewards/rejected": -0.7048042416572571, + "step": 5890 + }, + { + "epoch": 0.77, + "learning_rate": 7.507002186539147e-07, + "logits/chosen": -2.395439624786377, + "logits/rejected": -2.5001044273376465, + "logps/chosen": -360.6567077636719, + "logps/rejected": -345.16259765625, + "loss": 0.6263, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.28635045886039734, + "rewards/margins": 0.23526303470134735, + "rewards/rejected": -0.5216134786605835, + "step": 5900 + }, + { + "epoch": 0.77, + "learning_rate": 7.425593897573216e-07, + "logits/chosen": -2.476325511932373, + "logits/rejected": -2.4704856872558594, + "logps/chosen": -365.66754150390625, + "logps/rejected": -364.56768798828125, + "loss": 0.595, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.23280945420265198, + "rewards/margins": 0.36780303716659546, + "rewards/rejected": -0.6006125211715698, + "step": 5910 + }, + { + "epoch": 0.77, + "learning_rate": 7.344552368299088e-07, + "logits/chosen": -2.561272621154785, + "logits/rejected": -2.5794219970703125, + "logps/chosen": -313.61297607421875, + "logps/rejected": -329.70770263671875, + "loss": 0.5875, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.26516374945640564, + "rewards/margins": 0.23184308409690857, + "rewards/rejected": -0.4970068037509918, + "step": 5920 + }, + { + "epoch": 0.78, + "learning_rate": 7.26387928996973e-07, + "logits/chosen": -2.4242825508117676, + "logits/rejected": -2.4046523571014404, + "logps/chosen": -290.2831115722656, + "logps/rejected": -275.2210998535156, + "loss": 0.5484, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.41711360216140747, + "rewards/margins": 0.27641454339027405, + "rewards/rejected": -0.6935281157493591, + "step": 5930 + }, + { + "epoch": 0.78, + "learning_rate": 7.183576346148899e-07, + "logits/chosen": -2.6386637687683105, + "logits/rejected": -2.579282283782959, + "logps/chosen": -284.7298278808594, + "logps/rejected": -318.6064758300781, + "loss": 0.5745, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.2168664038181305, + "rewards/margins": 0.37781259417533875, + "rewards/rejected": -0.594679057598114, + "step": 5940 + }, + { + "epoch": 0.78, + "learning_rate": 7.103645212676044e-07, + "logits/chosen": -2.486691951751709, + "logits/rejected": -2.5294435024261475, + "logps/chosen": -260.233154296875, + "logps/rejected": -269.3633728027344, + "loss": 0.5684, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12218284606933594, + "rewards/margins": 0.41997766494750977, + "rewards/rejected": -0.5421606302261353, + "step": 5950 + }, + { + "epoch": 0.78, + "learning_rate": 7.024087557631318e-07, + "logits/chosen": -2.462345838546753, + "logits/rejected": -2.4638350009918213, + "logps/chosen": -264.4425048828125, + "logps/rejected": -300.456787109375, + "loss": 0.6159, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3224496841430664, + "rewards/margins": 0.18295326828956604, + "rewards/rejected": -0.5054029226303101, + "step": 5960 + }, + { + "epoch": 0.78, + "learning_rate": 6.944905041300739e-07, + "logits/chosen": -2.4791269302368164, + "logits/rejected": -2.4125843048095703, + "logps/chosen": -333.6778869628906, + "logps/rejected": -301.9044189453125, + "loss": 0.5248, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21129567921161652, + "rewards/margins": 0.4146009385585785, + "rewards/rejected": -0.6258966326713562, + "step": 5970 + }, + { + "epoch": 0.78, + "learning_rate": 6.866099316141606e-07, + "logits/chosen": -2.6854865550994873, + "logits/rejected": -2.6372344493865967, + "logps/chosen": -300.1522216796875, + "logps/rejected": -308.7620544433594, + "loss": 0.5992, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.29093360900878906, + "rewards/margins": 0.2515958249568939, + "rewards/rejected": -0.5425294637680054, + "step": 5980 + }, + { + "epoch": 0.78, + "learning_rate": 6.787672026747946e-07, + "logits/chosen": -2.477449893951416, + "logits/rejected": -2.531902313232422, + "logps/chosen": -295.63287353515625, + "logps/rejected": -329.295654296875, + "loss": 0.617, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.28388139605522156, + "rewards/margins": 0.18577614426612854, + "rewards/rejected": -0.4696575105190277, + "step": 5990 + }, + { + "epoch": 0.79, + "learning_rate": 6.709624809816223e-07, + "logits/chosen": -2.5245113372802734, + "logits/rejected": -2.5861029624938965, + "logps/chosen": -321.34124755859375, + "logps/rejected": -345.78643798828125, + "loss": 0.5814, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1564144492149353, + "rewards/margins": 0.43052688241004944, + "rewards/rejected": -0.5869413614273071, + "step": 6000 + }, + { + "epoch": 0.79, + "eval_logits/chosen": -2.5504660606384277, + "eval_logits/rejected": -2.520192861557007, + "eval_logps/chosen": -308.9028625488281, + "eval_logps/rejected": -305.8824462890625, + "eval_loss": 0.586705207824707, + "eval_rewards/accuracies": 0.7080000042915344, + "eval_rewards/chosen": -0.24183543026447296, + "eval_rewards/margins": 0.30403584241867065, + "eval_rewards/rejected": -0.5458711981773376, + "eval_runtime": 469.9296, + "eval_samples_per_second": 4.256, + "eval_steps_per_second": 1.064, + "step": 6000 + }, + { + "epoch": 0.79, + "learning_rate": 6.6319592941112e-07, + "logits/chosen": -2.6814560890197754, + "logits/rejected": -2.6334147453308105, + "logps/chosen": -373.9043273925781, + "logps/rejected": -349.0970153808594, + "loss": 0.5531, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.13861213624477386, + "rewards/margins": 0.39211505651474, + "rewards/rejected": -0.5307271480560303, + "step": 6010 + }, + { + "epoch": 0.79, + "learning_rate": 6.554677100431927e-07, + "logits/chosen": -2.579408645629883, + "logits/rejected": -2.473529100418091, + "logps/chosen": -315.31787109375, + "logps/rejected": -308.83428955078125, + "loss": 0.6099, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2672146260738373, + "rewards/margins": 0.30611953139305115, + "rewards/rejected": -0.5733340978622437, + "step": 6020 + }, + { + "epoch": 0.79, + "learning_rate": 6.4777798415779e-07, + "logits/chosen": -2.5432662963867188, + "logits/rejected": -2.6698403358459473, + "logps/chosen": -305.22607421875, + "logps/rejected": -312.6539001464844, + "loss": 0.5846, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.24335560202598572, + "rewards/margins": 0.3419436514377594, + "rewards/rejected": -0.5852991938591003, + "step": 6030 + }, + { + "epoch": 0.79, + "learning_rate": 6.401269122315451e-07, + "logits/chosen": -2.3377437591552734, + "logits/rejected": -2.317566394805908, + "logps/chosen": -245.70132446289062, + "logps/rejected": -307.01837158203125, + "loss": 0.5634, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.21264919638633728, + "rewards/margins": 0.40377864241600037, + "rewards/rejected": -0.6164278984069824, + "step": 6040 + }, + { + "epoch": 0.79, + "learning_rate": 6.325146539344196e-07, + "logits/chosen": -2.454629898071289, + "logits/rejected": -2.4771924018859863, + "logps/chosen": -363.04827880859375, + "logps/rejected": -350.6267395019531, + "loss": 0.5681, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.37394410371780396, + "rewards/margins": 0.2525387406349182, + "rewards/rejected": -0.6264828443527222, + "step": 6050 + }, + { + "epoch": 0.79, + "learning_rate": 6.249413681263782e-07, + "logits/chosen": -2.641986846923828, + "logits/rejected": -2.6534454822540283, + "logps/chosen": -259.7637023925781, + "logps/rejected": -276.17962646484375, + "loss": 0.5925, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.19203992187976837, + "rewards/margins": 0.2957462668418884, + "rewards/rejected": -0.4877861440181732, + "step": 6060 + }, + { + "epoch": 0.79, + "learning_rate": 6.174072128540686e-07, + "logits/chosen": -2.4908456802368164, + "logits/rejected": -2.480006217956543, + "logps/chosen": -293.46746826171875, + "logps/rejected": -304.4007568359375, + "loss": 0.5475, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1637772023677826, + "rewards/margins": 0.4057283401489258, + "rewards/rejected": -0.569505512714386, + "step": 6070 + }, + { + "epoch": 0.8, + "learning_rate": 6.099123453475245e-07, + "logits/chosen": -2.514127016067505, + "logits/rejected": -2.460484504699707, + "logps/chosen": -310.1918640136719, + "logps/rejected": -283.559326171875, + "loss": 0.5942, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2885943055152893, + "rewards/margins": 0.2289300411939621, + "rewards/rejected": -0.5175243616104126, + "step": 6080 + }, + { + "epoch": 0.8, + "learning_rate": 6.024569220168836e-07, + "logits/chosen": -2.5178470611572266, + "logits/rejected": -2.5889010429382324, + "logps/chosen": -299.262451171875, + "logps/rejected": -286.537109375, + "loss": 0.5779, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.23814013600349426, + "rewards/margins": 0.2815389335155487, + "rewards/rejected": -0.519679069519043, + "step": 6090 + }, + { + "epoch": 0.8, + "learning_rate": 5.950410984491268e-07, + "logits/chosen": -2.7100613117218018, + "logits/rejected": -2.7072670459747314, + "logps/chosen": -364.88330078125, + "logps/rejected": -345.30145263671875, + "loss": 0.5909, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.23071260750293732, + "rewards/margins": 0.32682785391807556, + "rewards/rejected": -0.5575404763221741, + "step": 6100 + }, + { + "epoch": 0.8, + "learning_rate": 5.876650294048262e-07, + "logits/chosen": -2.4635324478149414, + "logits/rejected": -2.5075535774230957, + "logps/chosen": -290.86737060546875, + "logps/rejected": -348.6126403808594, + "loss": 0.5878, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21459011733531952, + "rewards/margins": 0.36013904213905334, + "rewards/rejected": -0.5747290849685669, + "step": 6110 + }, + { + "epoch": 0.8, + "learning_rate": 5.8032886881492e-07, + "logits/chosen": -2.7076010704040527, + "logits/rejected": -2.550320863723755, + "logps/chosen": -326.60009765625, + "logps/rejected": -403.6198425292969, + "loss": 0.6187, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.31724706292152405, + "rewards/margins": 0.22199532389640808, + "rewards/rejected": -0.5392423868179321, + "step": 6120 + }, + { + "epoch": 0.8, + "learning_rate": 5.730327697774988e-07, + "logits/chosen": -2.5117106437683105, + "logits/rejected": -2.506225347518921, + "logps/chosen": -316.81219482421875, + "logps/rejected": -331.5554504394531, + "loss": 0.6249, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2280002385377884, + "rewards/margins": 0.21184459328651428, + "rewards/rejected": -0.43984484672546387, + "step": 6130 + }, + { + "epoch": 0.8, + "learning_rate": 5.657768845546068e-07, + "logits/chosen": -2.4038338661193848, + "logits/rejected": -2.5388214588165283, + "logps/chosen": -307.8619079589844, + "logps/rejected": -383.5082702636719, + "loss": 0.605, + "rewards/accuracies": 0.5249999761581421, + "rewards/chosen": -0.2060864418745041, + "rewards/margins": 0.13893654942512512, + "rewards/rejected": -0.345022976398468, + "step": 6140 + }, + { + "epoch": 0.8, + "learning_rate": 5.585613645690713e-07, + "logits/chosen": -2.517221450805664, + "logits/rejected": -2.4837019443511963, + "logps/chosen": -259.75616455078125, + "logps/rejected": -294.917236328125, + "loss": 0.5567, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.29537954926490784, + "rewards/margins": 0.40257397294044495, + "rewards/rejected": -0.6979535818099976, + "step": 6150 + }, + { + "epoch": 0.81, + "learning_rate": 5.513863604013355e-07, + "logits/chosen": -2.53302001953125, + "logits/rejected": -2.406219720840454, + "logps/chosen": -396.5958251953125, + "logps/rejected": -334.18170166015625, + "loss": 0.5657, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.08400538563728333, + "rewards/margins": 0.5109080076217651, + "rewards/rejected": -0.5949134230613708, + "step": 6160 + }, + { + "epoch": 0.81, + "learning_rate": 5.442520217863215e-07, + "logits/chosen": -2.689875364303589, + "logits/rejected": -2.602809190750122, + "logps/chosen": -288.01361083984375, + "logps/rejected": -299.61407470703125, + "loss": 0.5664, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.2592290937900543, + "rewards/margins": 0.291963666677475, + "rewards/rejected": -0.5511927604675293, + "step": 6170 + }, + { + "epoch": 0.81, + "learning_rate": 5.371584976103034e-07, + "logits/chosen": -2.6083056926727295, + "logits/rejected": -2.544882297515869, + "logps/chosen": -305.5341796875, + "logps/rejected": -331.2020568847656, + "loss": 0.5763, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.14928746223449707, + "rewards/margins": 0.43168264627456665, + "rewards/rejected": -0.5809701681137085, + "step": 6180 + }, + { + "epoch": 0.81, + "learning_rate": 5.301059359077987e-07, + "logits/chosen": -2.5736923217773438, + "logits/rejected": -2.5212347507476807, + "logps/chosen": -296.933349609375, + "logps/rejected": -293.7825622558594, + "loss": 0.5969, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20169103145599365, + "rewards/margins": 0.3371420204639435, + "rewards/rejected": -0.5388330221176147, + "step": 6190 + }, + { + "epoch": 0.81, + "learning_rate": 5.230944838584806e-07, + "logits/chosen": -2.5412113666534424, + "logits/rejected": -2.5593161582946777, + "logps/chosen": -317.4584655761719, + "logps/rejected": -338.98431396484375, + "loss": 0.5554, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.21396486461162567, + "rewards/margins": 0.44962701201438904, + "rewards/rejected": -0.6635918617248535, + "step": 6200 + }, + { + "epoch": 0.81, + "learning_rate": 5.161242877841083e-07, + "logits/chosen": -2.5664896965026855, + "logits/rejected": -2.5260627269744873, + "logps/chosen": -327.7658386230469, + "logps/rejected": -291.395263671875, + "loss": 0.5891, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.3413836359977722, + "rewards/margins": 0.1652899980545044, + "rewards/rejected": -0.5066736340522766, + "step": 6210 + }, + { + "epoch": 0.81, + "learning_rate": 5.091954931454682e-07, + "logits/chosen": -2.5482285022735596, + "logits/rejected": -2.5847091674804688, + "logps/chosen": -298.89947509765625, + "logps/rejected": -294.7454833984375, + "loss": 0.6506, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.22662201523780823, + "rewards/margins": 0.11685802787542343, + "rewards/rejected": -0.34348005056381226, + "step": 6220 + }, + { + "epoch": 0.82, + "learning_rate": 5.023082445393446e-07, + "logits/chosen": -2.6040854454040527, + "logits/rejected": -2.5687479972839355, + "logps/chosen": -270.38751220703125, + "logps/rejected": -280.75469970703125, + "loss": 0.5924, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.250823050737381, + "rewards/margins": 0.2815064787864685, + "rewards/rejected": -0.5323294997215271, + "step": 6230 + }, + { + "epoch": 0.82, + "learning_rate": 4.95462685695498e-07, + "logits/chosen": -2.544628620147705, + "logits/rejected": -2.547745704650879, + "logps/chosen": -323.119140625, + "logps/rejected": -350.0447998046875, + "loss": 0.5791, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.22325992584228516, + "rewards/margins": 0.27086228132247925, + "rewards/rejected": -0.4941222071647644, + "step": 6240 + }, + { + "epoch": 0.82, + "learning_rate": 4.88658959473666e-07, + "logits/chosen": -2.5207772254943848, + "logits/rejected": -2.5802581310272217, + "logps/chosen": -308.8432312011719, + "logps/rejected": -359.64288330078125, + "loss": 0.5673, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14637571573257446, + "rewards/margins": 0.41771894693374634, + "rewards/rejected": -0.5640946626663208, + "step": 6250 + }, + { + "epoch": 0.82, + "learning_rate": 4.818972078605821e-07, + "logits/chosen": -2.5128281116485596, + "logits/rejected": -2.457853317260742, + "logps/chosen": -299.13055419921875, + "logps/rejected": -281.8258361816406, + "loss": 0.6092, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.31060346961021423, + "rewards/margins": 0.19784040749073029, + "rewards/rejected": -0.5084439516067505, + "step": 6260 + }, + { + "epoch": 0.82, + "learning_rate": 4.7517757196701514e-07, + "logits/chosen": -2.509042263031006, + "logits/rejected": -2.427283525466919, + "logps/chosen": -240.78555297851562, + "logps/rejected": -285.3486633300781, + "loss": 0.6152, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.3734577000141144, + "rewards/margins": 0.23900923132896423, + "rewards/rejected": -0.6124669313430786, + "step": 6270 + }, + { + "epoch": 0.82, + "learning_rate": 4.6850019202482193e-07, + "logits/chosen": -2.512200117111206, + "logits/rejected": -2.500453472137451, + "logps/chosen": -273.4225158691406, + "logps/rejected": -334.0801696777344, + "loss": 0.5791, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.18041428923606873, + "rewards/margins": 0.3033163249492645, + "rewards/rejected": -0.48373061418533325, + "step": 6280 + }, + { + "epoch": 0.82, + "learning_rate": 4.618652073840188e-07, + "logits/chosen": -2.600834369659424, + "logits/rejected": -2.5488085746765137, + "logps/chosen": -323.1775817871094, + "logps/rejected": -324.8509216308594, + "loss": 0.5777, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.25158780813217163, + "rewards/margins": 0.28678256273269653, + "rewards/rejected": -0.5383704304695129, + "step": 6290 + }, + { + "epoch": 0.82, + "learning_rate": 4.5527275650987965e-07, + "logits/chosen": -2.522247552871704, + "logits/rejected": -2.547168254852295, + "logps/chosen": -266.584228515625, + "logps/rejected": -314.6968994140625, + "loss": 0.5841, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.25916263461112976, + "rewards/margins": 0.39021292328834534, + "rewards/rejected": -0.6493755578994751, + "step": 6300 + }, + { + "epoch": 0.83, + "learning_rate": 4.487229769800394e-07, + "logits/chosen": -2.5155081748962402, + "logits/rejected": -2.467679262161255, + "logps/chosen": -289.71258544921875, + "logps/rejected": -309.3189697265625, + "loss": 0.602, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.24935400485992432, + "rewards/margins": 0.3160592019557953, + "rewards/rejected": -0.5654131770133972, + "step": 6310 + }, + { + "epoch": 0.83, + "learning_rate": 4.422160054816285e-07, + "logits/chosen": -2.5616321563720703, + "logits/rejected": -2.6143195629119873, + "logps/chosen": -299.6599426269531, + "logps/rejected": -307.5855407714844, + "loss": 0.5826, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.254104882478714, + "rewards/margins": 0.33058133721351624, + "rewards/rejected": -0.5846861600875854, + "step": 6320 + }, + { + "epoch": 0.83, + "learning_rate": 4.35751977808416e-07, + "logits/chosen": -2.583028793334961, + "logits/rejected": -2.5282931327819824, + "logps/chosen": -372.45013427734375, + "logps/rejected": -405.28350830078125, + "loss": 0.5127, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.14416107535362244, + "rewards/margins": 0.551115870475769, + "rewards/rejected": -0.6952770352363586, + "step": 6330 + }, + { + "epoch": 0.83, + "learning_rate": 4.293310288579794e-07, + "logits/chosen": -2.588747262954712, + "logits/rejected": -2.5295772552490234, + "logps/chosen": -273.9884338378906, + "logps/rejected": -282.23907470703125, + "loss": 0.5938, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19281110167503357, + "rewards/margins": 0.29851463437080383, + "rewards/rejected": -0.4913257658481598, + "step": 6340 + }, + { + "epoch": 0.83, + "learning_rate": 4.2295329262888733e-07, + "logits/chosen": -2.4053893089294434, + "logits/rejected": -2.4115614891052246, + "logps/chosen": -342.3263854980469, + "logps/rejected": -388.1925048828125, + "loss": 0.5984, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3017456531524658, + "rewards/margins": 0.3246362805366516, + "rewards/rejected": -0.6263819932937622, + "step": 6350 + }, + { + "epoch": 0.83, + "learning_rate": 4.1661890221790316e-07, + "logits/chosen": -2.5638811588287354, + "logits/rejected": -2.525207996368408, + "logps/chosen": -340.7550964355469, + "logps/rejected": -313.72015380859375, + "loss": 0.5626, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.23626506328582764, + "rewards/margins": 0.39344462752342224, + "rewards/rejected": -0.6297096014022827, + "step": 6360 + }, + { + "epoch": 0.83, + "learning_rate": 4.103279898172072e-07, + "logits/chosen": -2.5897655487060547, + "logits/rejected": -2.541827917098999, + "logps/chosen": -323.07861328125, + "logps/rejected": -299.7076110839844, + "loss": 0.5254, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.22562608122825623, + "rewards/margins": 0.38975054025650024, + "rewards/rejected": -0.6153766512870789, + "step": 6370 + }, + { + "epoch": 0.83, + "learning_rate": 4.040806867116401e-07, + "logits/chosen": -2.6776676177978516, + "logits/rejected": -2.617333173751831, + "logps/chosen": -325.8877258300781, + "logps/rejected": -345.1810607910156, + "loss": 0.5894, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2222873866558075, + "rewards/margins": 0.29427021741867065, + "rewards/rejected": -0.5165575742721558, + "step": 6380 + }, + { + "epoch": 0.84, + "learning_rate": 3.978771232759615e-07, + "logits/chosen": -2.5058422088623047, + "logits/rejected": -2.6096720695495605, + "logps/chosen": -261.63604736328125, + "logps/rejected": -275.4947204589844, + "loss": 0.565, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17887890338897705, + "rewards/margins": 0.3644290268421173, + "rewards/rejected": -0.5433079600334167, + "step": 6390 + }, + { + "epoch": 0.84, + "learning_rate": 3.917174289721276e-07, + "logits/chosen": -2.387831211090088, + "logits/rejected": -2.41300630569458, + "logps/chosen": -294.5731506347656, + "logps/rejected": -296.50115966796875, + "loss": 0.5222, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1628241389989853, + "rewards/margins": 0.5051959753036499, + "rewards/rejected": -0.6680201292037964, + "step": 6400 + }, + { + "epoch": 0.84, + "learning_rate": 3.856017323465938e-07, + "logits/chosen": -2.6898727416992188, + "logits/rejected": -2.568655490875244, + "logps/chosen": -296.9098815917969, + "logps/rejected": -276.76495361328125, + "loss": 0.5912, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.3579789996147156, + "rewards/margins": 0.22693082690238953, + "rewards/rejected": -0.5849098563194275, + "step": 6410 + }, + { + "epoch": 0.84, + "learning_rate": 3.7953016102762695e-07, + "logits/chosen": -2.5304763317108154, + "logits/rejected": -2.507497549057007, + "logps/chosen": -321.936767578125, + "logps/rejected": -273.9273986816406, + "loss": 0.5651, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.1546071320772171, + "rewards/margins": 0.3859509229660034, + "rewards/rejected": -0.5405580401420593, + "step": 6420 + }, + { + "epoch": 0.84, + "learning_rate": 3.7350284172264493e-07, + "logits/chosen": -2.495654582977295, + "logits/rejected": -2.462907314300537, + "logps/chosen": -303.0787353515625, + "logps/rejected": -300.4488830566406, + "loss": 0.5997, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.30195024609565735, + "rewards/margins": 0.36328279972076416, + "rewards/rejected": -0.6652330160140991, + "step": 6430 + }, + { + "epoch": 0.84, + "learning_rate": 3.67519900215573e-07, + "logits/chosen": -2.623809814453125, + "logits/rejected": -2.636713743209839, + "logps/chosen": -305.46063232421875, + "logps/rejected": -310.02642822265625, + "loss": 0.5753, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.18001635372638702, + "rewards/margins": 0.3698897361755371, + "rewards/rejected": -0.5499060750007629, + "step": 6440 + }, + { + "epoch": 0.84, + "learning_rate": 3.615814613642174e-07, + "logits/chosen": -2.5246150493621826, + "logits/rejected": -2.5499939918518066, + "logps/chosen": -316.9003601074219, + "logps/rejected": -327.5944519042969, + "loss": 0.6326, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.28238850831985474, + "rewards/margins": 0.1322154551744461, + "rewards/rejected": -0.4146040380001068, + "step": 6450 + }, + { + "epoch": 0.85, + "learning_rate": 3.5568764909765795e-07, + "logits/chosen": -2.6263697147369385, + "logits/rejected": -2.6150715351104736, + "logps/chosen": -328.6458435058594, + "logps/rejected": -268.51361083984375, + "loss": 0.6494, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.15244045853614807, + "rewards/margins": 0.2182413786649704, + "rewards/rejected": -0.3706818222999573, + "step": 6460 + }, + { + "epoch": 0.85, + "learning_rate": 3.498385864136672e-07, + "logits/chosen": -2.556443691253662, + "logits/rejected": -2.5505318641662598, + "logps/chosen": -343.93829345703125, + "logps/rejected": -317.4654846191406, + "loss": 0.5867, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.25182396173477173, + "rewards/margins": 0.3008098006248474, + "rewards/rejected": -0.5526337027549744, + "step": 6470 + }, + { + "epoch": 0.85, + "learning_rate": 3.440343953761363e-07, + "logits/chosen": -2.4140422344207764, + "logits/rejected": -2.3615925312042236, + "logps/chosen": -269.6112976074219, + "logps/rejected": -271.3714294433594, + "loss": 0.5731, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1203186959028244, + "rewards/margins": 0.4729042649269104, + "rewards/rejected": -0.5932229161262512, + "step": 6480 + }, + { + "epoch": 0.85, + "learning_rate": 3.382751971125345e-07, + "logits/chosen": -2.5431666374206543, + "logits/rejected": -2.562822103500366, + "logps/chosen": -324.5274353027344, + "logps/rejected": -372.4595642089844, + "loss": 0.6288, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.18169930577278137, + "rewards/margins": 0.21922583878040314, + "rewards/rejected": -0.4009251594543457, + "step": 6490 + }, + { + "epoch": 0.85, + "learning_rate": 3.3256111181137753e-07, + "logits/chosen": -2.593381643295288, + "logits/rejected": -2.5534257888793945, + "logps/chosen": -278.71295166015625, + "logps/rejected": -332.32916259765625, + "loss": 0.6147, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.2595116198062897, + "rewards/margins": 0.2278112918138504, + "rewards/rejected": -0.4873228967189789, + "step": 6500 + }, + { + "epoch": 0.85, + "learning_rate": 3.2689225871971905e-07, + "logits/chosen": -2.5177037715911865, + "logits/rejected": -2.5049195289611816, + "logps/chosen": -307.04730224609375, + "logps/rejected": -303.7582702636719, + "loss": 0.6577, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.3698388636112213, + "rewards/margins": 0.12160823494195938, + "rewards/rejected": -0.4914471209049225, + "step": 6510 + }, + { + "epoch": 0.85, + "learning_rate": 3.2126875614066523e-07, + "logits/chosen": -2.610668420791626, + "logits/rejected": -2.5742154121398926, + "logps/chosen": -333.54290771484375, + "logps/rejected": -353.10052490234375, + "loss": 0.5332, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.18249043822288513, + "rewards/margins": 0.4640236794948578, + "rewards/rejected": -0.6465141773223877, + "step": 6520 + }, + { + "epoch": 0.85, + "learning_rate": 3.156907214309024e-07, + "logits/chosen": -2.494969367980957, + "logits/rejected": -2.5566751956939697, + "logps/chosen": -263.52508544921875, + "logps/rejected": -259.69818115234375, + "loss": 0.6364, + "rewards/accuracies": 0.4749999940395355, + "rewards/chosen": -0.2828473448753357, + "rewards/margins": 0.06948140263557434, + "rewards/rejected": -0.3523287773132324, + "step": 6530 + }, + { + "epoch": 0.86, + "learning_rate": 3.1015827099824923e-07, + "logits/chosen": -2.586190938949585, + "logits/rejected": -2.579084634780884, + "logps/chosen": -332.7265319824219, + "logps/rejected": -330.85589599609375, + "loss": 0.6042, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.14104333519935608, + "rewards/margins": 0.2698945701122284, + "rewards/rejected": -0.41093793511390686, + "step": 6540 + }, + { + "epoch": 0.86, + "learning_rate": 3.0467152029922926e-07, + "logits/chosen": -2.380070447921753, + "logits/rejected": -2.4754655361175537, + "logps/chosen": -284.60626220703125, + "logps/rejected": -284.9497985839844, + "loss": 0.5451, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1199837327003479, + "rewards/margins": 0.3513313829898834, + "rewards/rejected": -0.4713151454925537, + "step": 6550 + }, + { + "epoch": 0.86, + "learning_rate": 2.992305838366591e-07, + "logits/chosen": -2.6500418186187744, + "logits/rejected": -2.5698771476745605, + "logps/chosen": -240.611083984375, + "logps/rejected": -251.97286987304688, + "loss": 0.5578, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.20479588210582733, + "rewards/margins": 0.309184730052948, + "rewards/rejected": -0.5139805674552917, + "step": 6560 + }, + { + "epoch": 0.86, + "learning_rate": 2.938355751572583e-07, + "logits/chosen": -2.569633960723877, + "logits/rejected": -2.511861562728882, + "logps/chosen": -250.4905242919922, + "logps/rejected": -290.0649719238281, + "loss": 0.5421, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.14301282167434692, + "rewards/margins": 0.4621787965297699, + "rewards/rejected": -0.6051915884017944, + "step": 6570 + }, + { + "epoch": 0.86, + "learning_rate": 2.8848660684928307e-07, + "logits/chosen": -2.5781993865966797, + "logits/rejected": -2.5404911041259766, + "logps/chosen": -262.2217102050781, + "logps/rejected": -348.08282470703125, + "loss": 0.5624, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.24088053405284882, + "rewards/margins": 0.3800331652164459, + "rewards/rejected": -0.6209137439727783, + "step": 6580 + }, + { + "epoch": 0.86, + "learning_rate": 2.8318379054017383e-07, + "logits/chosen": -2.5964581966400146, + "logits/rejected": -2.445565700531006, + "logps/chosen": -320.2331848144531, + "logps/rejected": -380.0176696777344, + "loss": 0.5722, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.28656232357025146, + "rewards/margins": 0.2690221667289734, + "rewards/rejected": -0.5555844306945801, + "step": 6590 + }, + { + "epoch": 0.86, + "learning_rate": 2.779272368942246e-07, + "logits/chosen": -2.514643669128418, + "logits/rejected": -2.478142738342285, + "logps/chosen": -310.20135498046875, + "logps/rejected": -284.0975646972656, + "loss": 0.6536, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.3290698230266571, + "rewards/margins": 0.18459972739219666, + "rewards/rejected": -0.5136696100234985, + "step": 6600 + }, + { + "epoch": 0.86, + "learning_rate": 2.7271705561027986e-07, + "logits/chosen": -2.6962289810180664, + "logits/rejected": -2.6481692790985107, + "logps/chosen": -344.97625732421875, + "logps/rejected": -396.3194885253906, + "loss": 0.5275, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.14827750623226166, + "rewards/margins": 0.4757654070854187, + "rewards/rejected": -0.6240429282188416, + "step": 6610 + }, + { + "epoch": 0.87, + "learning_rate": 2.6755335541943677e-07, + "logits/chosen": -2.4567790031433105, + "logits/rejected": -2.4796199798583984, + "logps/chosen": -289.46539306640625, + "logps/rejected": -292.6962890625, + "loss": 0.5973, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.349151611328125, + "rewards/margins": 0.12624169886112213, + "rewards/rejected": -0.47539329528808594, + "step": 6620 + }, + { + "epoch": 0.87, + "learning_rate": 2.62436244082781e-07, + "logits/chosen": -2.536022424697876, + "logits/rejected": -2.5415444374084473, + "logps/chosen": -295.7829895019531, + "logps/rejected": -287.15252685546875, + "loss": 0.6223, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.29702284932136536, + "rewards/margins": 0.15536999702453613, + "rewards/rejected": -0.4523928165435791, + "step": 6630 + }, + { + "epoch": 0.87, + "learning_rate": 2.5736582838913836e-07, + "logits/chosen": -2.590641975402832, + "logits/rejected": -2.5924506187438965, + "logps/chosen": -281.2153625488281, + "logps/rejected": -286.9208984375, + "loss": 0.5854, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1338876485824585, + "rewards/margins": 0.29984337091445923, + "rewards/rejected": -0.43373093008995056, + "step": 6640 + }, + { + "epoch": 0.87, + "learning_rate": 2.5234221415284363e-07, + "logits/chosen": -2.6226627826690674, + "logits/rejected": -2.599210262298584, + "logps/chosen": -312.03912353515625, + "logps/rejected": -315.6419982910156, + "loss": 0.5975, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.24639169871807098, + "rewards/margins": 0.3372633457183838, + "rewards/rejected": -0.583655059337616, + "step": 6650 + }, + { + "epoch": 0.87, + "learning_rate": 2.4736550621153375e-07, + "logits/chosen": -2.5119102001190186, + "logits/rejected": -2.5432655811309814, + "logps/chosen": -354.1219177246094, + "logps/rejected": -350.56298828125, + "loss": 0.6117, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.26676005125045776, + "rewards/margins": 0.3218335211277008, + "rewards/rejected": -0.588593602180481, + "step": 6660 + }, + { + "epoch": 0.87, + "learning_rate": 2.424358084239609e-07, + "logits/chosen": -2.6142399311065674, + "logits/rejected": -2.713916301727295, + "logps/chosen": -411.18341064453125, + "logps/rejected": -339.60308837890625, + "loss": 0.6018, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.21380481123924255, + "rewards/margins": 0.25104427337646484, + "rewards/rejected": -0.464849054813385, + "step": 6670 + }, + { + "epoch": 0.87, + "learning_rate": 2.3755322366782158e-07, + "logits/chosen": -2.48545503616333, + "logits/rejected": -2.5491490364074707, + "logps/chosen": -332.91790771484375, + "logps/rejected": -318.849853515625, + "loss": 0.5744, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.24387934803962708, + "rewards/margins": 0.32038286328315735, + "rewards/rejected": -0.5642622709274292, + "step": 6680 + }, + { + "epoch": 0.88, + "learning_rate": 2.3271785383761431e-07, + "logits/chosen": -2.4943230152130127, + "logits/rejected": -2.4346394538879395, + "logps/chosen": -380.49920654296875, + "logps/rejected": -364.5450134277344, + "loss": 0.6466, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2653929591178894, + "rewards/margins": 0.2565198540687561, + "rewards/rejected": -0.5219127535820007, + "step": 6690 + }, + { + "epoch": 0.88, + "learning_rate": 2.2792979984250978e-07, + "logits/chosen": -2.5065841674804688, + "logits/rejected": -2.488462448120117, + "logps/chosen": -316.1371765136719, + "logps/rejected": -299.7496643066406, + "loss": 0.5848, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.21795110404491425, + "rewards/margins": 0.313754141330719, + "rewards/rejected": -0.5317052602767944, + "step": 6700 + }, + { + "epoch": 0.88, + "learning_rate": 2.231891616042453e-07, + "logits/chosen": -2.511505365371704, + "logits/rejected": -2.5903966426849365, + "logps/chosen": -285.35870361328125, + "logps/rejected": -365.3995361328125, + "loss": 0.6514, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.26048222184181213, + "rewards/margins": 0.2787513732910156, + "rewards/rejected": -0.5392335653305054, + "step": 6710 + }, + { + "epoch": 0.88, + "learning_rate": 2.1849603805504328e-07, + "logits/chosen": -2.441913604736328, + "logits/rejected": -2.460020065307617, + "logps/chosen": -327.166015625, + "logps/rejected": -331.92279052734375, + "loss": 0.5754, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2605854868888855, + "rewards/margins": 0.294552743434906, + "rewards/rejected": -0.5551381707191467, + "step": 6720 + }, + { + "epoch": 0.88, + "learning_rate": 2.1385052713554066e-07, + "logits/chosen": -2.595977306365967, + "logits/rejected": -2.589634418487549, + "logps/chosen": -242.5916748046875, + "logps/rejected": -287.7140197753906, + "loss": 0.6079, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.36479222774505615, + "rewards/margins": 0.20024845004081726, + "rewards/rejected": -0.5650407075881958, + "step": 6730 + }, + { + "epoch": 0.88, + "learning_rate": 2.0925272579274873e-07, + "logits/chosen": -2.63620662689209, + "logits/rejected": -2.611586093902588, + "logps/chosen": -342.5591735839844, + "logps/rejected": -348.40460205078125, + "loss": 0.5766, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21620401740074158, + "rewards/margins": 0.3205227553844452, + "rewards/rejected": -0.5367268323898315, + "step": 6740 + }, + { + "epoch": 0.88, + "learning_rate": 2.047027299780302e-07, + "logits/chosen": -2.6527209281921387, + "logits/rejected": -2.7160065174102783, + "logps/chosen": -344.5036315917969, + "logps/rejected": -357.87042236328125, + "loss": 0.5907, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.3077239990234375, + "rewards/margins": 0.29998379945755005, + "rewards/rejected": -0.6077077984809875, + "step": 6750 + }, + { + "epoch": 0.88, + "learning_rate": 2.0020063464509492e-07, + "logits/chosen": -2.5766146183013916, + "logits/rejected": -2.5696825981140137, + "logps/chosen": -284.848388671875, + "logps/rejected": -241.0084991455078, + "loss": 0.5643, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12015555053949356, + "rewards/margins": 0.4285253584384918, + "rewards/rejected": -0.5486809015274048, + "step": 6760 + }, + { + "epoch": 0.89, + "learning_rate": 1.957465337480191e-07, + "logits/chosen": -2.433406352996826, + "logits/rejected": -2.4728941917419434, + "logps/chosen": -270.2312316894531, + "logps/rejected": -258.283447265625, + "loss": 0.5863, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16889183223247528, + "rewards/margins": 0.33343085646629333, + "rewards/rejected": -0.5023227334022522, + "step": 6770 + }, + { + "epoch": 0.89, + "learning_rate": 1.9134052023928622e-07, + "logits/chosen": -2.490142345428467, + "logits/rejected": -2.5064761638641357, + "logps/chosen": -260.85211181640625, + "logps/rejected": -286.789306640625, + "loss": 0.6311, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.26439350843429565, + "rewards/margins": 0.2775447964668274, + "rewards/rejected": -0.541938304901123, + "step": 6780 + }, + { + "epoch": 0.89, + "learning_rate": 1.8698268606784392e-07, + "logits/chosen": -2.5506930351257324, + "logits/rejected": -2.472028970718384, + "logps/chosen": -279.47833251953125, + "logps/rejected": -317.755126953125, + "loss": 0.5259, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1397266685962677, + "rewards/margins": 0.4272634983062744, + "rewards/rejected": -0.5669901371002197, + "step": 6790 + }, + { + "epoch": 0.89, + "learning_rate": 1.826731221771866e-07, + "logits/chosen": -2.5460689067840576, + "logits/rejected": -2.5425117015838623, + "logps/chosen": -354.0918884277344, + "logps/rejected": -331.4277038574219, + "loss": 0.5989, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21315228939056396, + "rewards/margins": 0.4096096456050873, + "rewards/rejected": -0.6227619647979736, + "step": 6800 + }, + { + "epoch": 0.89, + "learning_rate": 1.7841191850345967e-07, + "logits/chosen": -2.4992847442626953, + "logits/rejected": -2.553622245788574, + "logps/chosen": -304.04803466796875, + "logps/rejected": -341.6429138183594, + "loss": 0.5936, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.27363067865371704, + "rewards/margins": 0.3625665307044983, + "rewards/rejected": -0.6361972093582153, + "step": 6810 + }, + { + "epoch": 0.89, + "learning_rate": 1.7419916397357905e-07, + "logits/chosen": -2.529766798019409, + "logits/rejected": -2.4962515830993652, + "logps/chosen": -255.84994506835938, + "logps/rejected": -275.2969055175781, + "loss": 0.5974, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2806466221809387, + "rewards/margins": 0.21514853835105896, + "rewards/rejected": -0.49579519033432007, + "step": 6820 + }, + { + "epoch": 0.89, + "learning_rate": 1.700349465033782e-07, + "logits/chosen": -2.6027166843414307, + "logits/rejected": -2.534972906112671, + "logps/chosen": -345.79766845703125, + "logps/rejected": -291.8948669433594, + "loss": 0.6274, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.36087167263031006, + "rewards/margins": 0.25738582015037537, + "rewards/rejected": -0.618257462978363, + "step": 6830 + }, + { + "epoch": 0.9, + "learning_rate": 1.6591935299577227e-07, + "logits/chosen": -2.5771901607513428, + "logits/rejected": -2.568636178970337, + "logps/chosen": -335.08758544921875, + "logps/rejected": -357.94329833984375, + "loss": 0.5739, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.2538537085056305, + "rewards/margins": 0.3991164565086365, + "rewards/rejected": -0.6529701948165894, + "step": 6840 + }, + { + "epoch": 0.9, + "learning_rate": 1.6185246933894338e-07, + "logits/chosen": -2.544599771499634, + "logits/rejected": -2.4817824363708496, + "logps/chosen": -338.9317932128906, + "logps/rejected": -354.46575927734375, + "loss": 0.5834, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.35685110092163086, + "rewards/margins": 0.30958980321884155, + "rewards/rejected": -0.6664408445358276, + "step": 6850 + }, + { + "epoch": 0.9, + "learning_rate": 1.5783438040455097e-07, + "logits/chosen": -2.5295493602752686, + "logits/rejected": -2.6433188915252686, + "logps/chosen": -347.1949768066406, + "logps/rejected": -312.98980712890625, + "loss": 0.5991, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3010823130607605, + "rewards/margins": 0.13934998214244843, + "rewards/rejected": -0.4404323101043701, + "step": 6860 + }, + { + "epoch": 0.9, + "learning_rate": 1.538651700459576e-07, + "logits/chosen": -2.6250827312469482, + "logits/rejected": -2.590571403503418, + "logps/chosen": -387.9285583496094, + "logps/rejected": -334.91522216796875, + "loss": 0.6188, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.4069536328315735, + "rewards/margins": 0.23420529067516327, + "rewards/rejected": -0.641158938407898, + "step": 6870 + }, + { + "epoch": 0.9, + "learning_rate": 1.4994492109648151e-07, + "logits/chosen": -2.4459919929504395, + "logits/rejected": -2.3544578552246094, + "logps/chosen": -252.25808715820312, + "logps/rejected": -341.5327453613281, + "loss": 0.5787, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.20545756816864014, + "rewards/margins": 0.2764742970466614, + "rewards/rejected": -0.4819318652153015, + "step": 6880 + }, + { + "epoch": 0.9, + "learning_rate": 1.4607371536766695e-07, + "logits/chosen": -2.5497701168060303, + "logits/rejected": -2.6141464710235596, + "logps/chosen": -268.9443359375, + "logps/rejected": -308.8302001953125, + "loss": 0.6344, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.32575878500938416, + "rewards/margins": 0.21937373280525208, + "rewards/rejected": -0.5451325178146362, + "step": 6890 + }, + { + "epoch": 0.9, + "learning_rate": 1.4225163364757655e-07, + "logits/chosen": -2.6052086353302, + "logits/rejected": -2.554774522781372, + "logps/chosen": -365.8276062011719, + "logps/rejected": -333.3267517089844, + "loss": 0.5578, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.2463061809539795, + "rewards/margins": 0.2694723904132843, + "rewards/rejected": -0.5157786011695862, + "step": 6900 + }, + { + "epoch": 0.9, + "learning_rate": 1.3847875569910462e-07, + "logits/chosen": -2.527517080307007, + "logits/rejected": -2.5814757347106934, + "logps/chosen": -260.64996337890625, + "logps/rejected": -265.3381652832031, + "loss": 0.6107, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.3884786367416382, + "rewards/margins": 0.19082988798618317, + "rewards/rejected": -0.5793085098266602, + "step": 6910 + }, + { + "epoch": 0.91, + "learning_rate": 1.3475516025831552e-07, + "logits/chosen": -2.553699493408203, + "logits/rejected": -2.5805094242095947, + "logps/chosen": -235.4549102783203, + "logps/rejected": -278.7104797363281, + "loss": 0.5879, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.24513201415538788, + "rewards/margins": 0.22519740462303162, + "rewards/rejected": -0.4703294336795807, + "step": 6920 + }, + { + "epoch": 0.91, + "learning_rate": 1.310809250327974e-07, + "logits/chosen": -2.5498898029327393, + "logits/rejected": -2.5694942474365234, + "logps/chosen": -260.2915344238281, + "logps/rejected": -301.33331298828125, + "loss": 0.5541, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2523956000804901, + "rewards/margins": 0.3117953836917877, + "rewards/rejected": -0.5641909837722778, + "step": 6930 + }, + { + "epoch": 0.91, + "learning_rate": 1.2745612670004153e-07, + "logits/chosen": -2.6088833808898926, + "logits/rejected": -2.4972448348999023, + "logps/chosen": -282.6246032714844, + "logps/rejected": -294.3809509277344, + "loss": 0.5616, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.30884820222854614, + "rewards/margins": 0.40127071738243103, + "rewards/rejected": -0.7101189494132996, + "step": 6940 + }, + { + "epoch": 0.91, + "learning_rate": 1.2388084090584395e-07, + "logits/chosen": -2.553621530532837, + "logits/rejected": -2.5755157470703125, + "logps/chosen": -285.8392333984375, + "logps/rejected": -273.127685546875, + "loss": 0.5947, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.21612191200256348, + "rewards/margins": 0.37774962186813354, + "rewards/rejected": -0.5938714742660522, + "step": 6950 + }, + { + "epoch": 0.91, + "learning_rate": 1.2035514226272305e-07, + "logits/chosen": -2.54499888420105, + "logits/rejected": -2.6355156898498535, + "logps/chosen": -309.70703125, + "logps/rejected": -313.765869140625, + "loss": 0.6071, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.29586124420166016, + "rewards/margins": 0.153177410364151, + "rewards/rejected": -0.4490385949611664, + "step": 6960 + }, + { + "epoch": 0.91, + "learning_rate": 1.1687910434836607e-07, + "logits/chosen": -2.36899995803833, + "logits/rejected": -2.435290575027466, + "logps/chosen": -256.7616271972656, + "logps/rejected": -355.43634033203125, + "loss": 0.5517, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.23554377257823944, + "rewards/margins": 0.4043079912662506, + "rewards/rejected": -0.6398516893386841, + "step": 6970 + }, + { + "epoch": 0.91, + "learning_rate": 1.1345279970409128e-07, + "logits/chosen": -2.604055881500244, + "logits/rejected": -2.6000876426696777, + "logps/chosen": -293.7242431640625, + "logps/rejected": -350.16375732421875, + "loss": 0.6043, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.23609530925750732, + "rewards/margins": 0.22831125557422638, + "rewards/rejected": -0.4644065499305725, + "step": 6980 + }, + { + "epoch": 0.91, + "learning_rate": 1.1007629983333629e-07, + "logits/chosen": -2.3824760913848877, + "logits/rejected": -2.38716983795166, + "logps/chosen": -292.387939453125, + "logps/rejected": -310.6404724121094, + "loss": 0.5431, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.18008548021316528, + "rewards/margins": 0.550257682800293, + "rewards/rejected": -0.7303431630134583, + "step": 6990 + }, + { + "epoch": 0.92, + "learning_rate": 1.067496752001626e-07, + "logits/chosen": -2.5673344135284424, + "logits/rejected": -2.538301467895508, + "logps/chosen": -250.818115234375, + "logps/rejected": -313.76220703125, + "loss": 0.5811, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2165127545595169, + "rewards/margins": 0.34249112010002136, + "rewards/rejected": -0.5590038299560547, + "step": 7000 + }, + { + "epoch": 0.92, + "learning_rate": 1.0347299522778909e-07, + "logits/chosen": -2.5746231079101562, + "logits/rejected": -2.670893430709839, + "logps/chosen": -262.069580078125, + "logps/rejected": -300.20928955078125, + "loss": 0.6302, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.27484551072120667, + "rewards/margins": 0.19697365164756775, + "rewards/rejected": -0.4718192219734192, + "step": 7010 + }, + { + "epoch": 0.92, + "learning_rate": 1.0024632829713971e-07, + "logits/chosen": -2.5796866416931152, + "logits/rejected": -2.5593037605285645, + "logps/chosen": -247.50619506835938, + "logps/rejected": -334.5338439941406, + "loss": 0.5687, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16089312732219696, + "rewards/margins": 0.3748016953468323, + "rewards/rejected": -0.5356948375701904, + "step": 7020 + }, + { + "epoch": 0.92, + "learning_rate": 9.706974174541889e-08, + "logits/chosen": -2.612639904022217, + "logits/rejected": -2.5660667419433594, + "logps/chosen": -328.50079345703125, + "logps/rejected": -310.83734130859375, + "loss": 0.587, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2703118920326233, + "rewards/margins": 0.21857671439647675, + "rewards/rejected": -0.48888859152793884, + "step": 7030 + }, + { + "epoch": 0.92, + "learning_rate": 9.39433018647043e-08, + "logits/chosen": -2.5860233306884766, + "logits/rejected": -2.4884932041168213, + "logps/chosen": -287.6629333496094, + "logps/rejected": -283.45001220703125, + "loss": 0.5869, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.26323238015174866, + "rewards/margins": 0.3591728210449219, + "rewards/rejected": -0.6224051713943481, + "step": 7040 + }, + { + "epoch": 0.92, + "learning_rate": 9.086707390056543e-08, + "logits/chosen": -2.6537437438964844, + "logits/rejected": -2.6398043632507324, + "logps/chosen": -323.03753662109375, + "logps/rejected": -351.5426330566406, + "loss": 0.6427, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3780897259712219, + "rewards/margins": 0.2068289965391159, + "rewards/rejected": -0.584918737411499, + "step": 7050 + }, + { + "epoch": 0.92, + "learning_rate": 8.784112205070083e-08, + "logits/chosen": -2.5830557346343994, + "logits/rejected": -2.62549090385437, + "logps/chosen": -274.672119140625, + "logps/rejected": -305.0771484375, + "loss": 0.5494, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.09539235383272171, + "rewards/margins": 0.4771255850791931, + "rewards/rejected": -0.572517991065979, + "step": 7060 + }, + { + "epoch": 0.93, + "learning_rate": 8.486550946359779e-08, + "logits/chosen": -2.622972011566162, + "logits/rejected": -2.623706102371216, + "logps/chosen": -286.2050476074219, + "logps/rejected": -285.8695373535156, + "loss": 0.6466, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.33009567856788635, + "rewards/margins": 0.13764071464538574, + "rewards/rejected": -0.4677363336086273, + "step": 7070 + }, + { + "epoch": 0.93, + "learning_rate": 8.194029823721556e-08, + "logits/chosen": -2.66804575920105, + "logits/rejected": -2.6642093658447266, + "logps/chosen": -335.0277099609375, + "logps/rejected": -316.1394348144531, + "loss": 0.6425, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.2565035820007324, + "rewards/margins": 0.15874430537223816, + "rewards/rejected": -0.4152478575706482, + "step": 7080 + }, + { + "epoch": 0.93, + "learning_rate": 7.906554941768896e-08, + "logits/chosen": -2.480766534805298, + "logits/rejected": -2.4899449348449707, + "logps/chosen": -271.68438720703125, + "logps/rejected": -331.60064697265625, + "loss": 0.607, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.17925897240638733, + "rewards/margins": 0.2871427536010742, + "rewards/rejected": -0.46640172600746155, + "step": 7090 + }, + { + "epoch": 0.93, + "learning_rate": 7.624132299805575e-08, + "logits/chosen": -2.6367640495300293, + "logits/rejected": -2.642698287963867, + "logps/chosen": -314.950927734375, + "logps/rejected": -329.8536071777344, + "loss": 0.5392, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.128908172249794, + "rewards/margins": 0.4371621608734131, + "rewards/rejected": -0.5660702586174011, + "step": 7100 + }, + { + "epoch": 0.93, + "learning_rate": 7.346767791700127e-08, + "logits/chosen": -2.549582004547119, + "logits/rejected": -2.59175443649292, + "logps/chosen": -292.8898010253906, + "logps/rejected": -312.5947265625, + "loss": 0.5883, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.26378917694091797, + "rewards/margins": 0.34494879841804504, + "rewards/rejected": -0.6087379455566406, + "step": 7110 + }, + { + "epoch": 0.93, + "learning_rate": 7.07446720576327e-08, + "logits/chosen": -2.580864667892456, + "logits/rejected": -2.5099356174468994, + "logps/chosen": -319.59539794921875, + "logps/rejected": -322.5418395996094, + "loss": 0.5841, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.23274675011634827, + "rewards/margins": 0.3921774625778198, + "rewards/rejected": -0.6249242424964905, + "step": 7120 + }, + { + "epoch": 0.93, + "learning_rate": 6.807236224626701e-08, + "logits/chosen": -2.480311155319214, + "logits/rejected": -2.445802688598633, + "logps/chosen": -252.0524444580078, + "logps/rejected": -291.4988098144531, + "loss": 0.5848, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.313507616519928, + "rewards/margins": 0.20375177264213562, + "rewards/rejected": -0.517259418964386, + "step": 7130 + }, + { + "epoch": 0.93, + "learning_rate": 6.545080425124888e-08, + "logits/chosen": -2.58805251121521, + "logits/rejected": -2.58807373046875, + "logps/chosen": -279.99005126953125, + "logps/rejected": -277.5595397949219, + "loss": 0.6408, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2955562174320221, + "rewards/margins": 0.27668994665145874, + "rewards/rejected": -0.5722461938858032, + "step": 7140 + }, + { + "epoch": 0.94, + "learning_rate": 6.288005278178382e-08, + "logits/chosen": -2.661332130432129, + "logits/rejected": -2.63032603263855, + "logps/chosen": -300.8346252441406, + "logps/rejected": -303.6176452636719, + "loss": 0.6006, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1539689600467682, + "rewards/margins": 0.30857062339782715, + "rewards/rejected": -0.4625396132469177, + "step": 7150 + }, + { + "epoch": 0.94, + "learning_rate": 6.036016148679825e-08, + "logits/chosen": -2.4713082313537598, + "logits/rejected": -2.405475616455078, + "logps/chosen": -299.3686218261719, + "logps/rejected": -321.78045654296875, + "loss": 0.567, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.3375488221645355, + "rewards/margins": 0.2744831442832947, + "rewards/rejected": -0.6120319962501526, + "step": 7160 + }, + { + "epoch": 0.94, + "learning_rate": 5.7891182953819235e-08, + "logits/chosen": -2.4739296436309814, + "logits/rejected": -2.4340744018554688, + "logps/chosen": -314.0517883300781, + "logps/rejected": -305.4825744628906, + "loss": 0.6257, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.3760947585105896, + "rewards/margins": 0.10057481378316879, + "rewards/rejected": -0.4766695499420166, + "step": 7170 + }, + { + "epoch": 0.94, + "learning_rate": 5.547316870787689e-08, + "logits/chosen": -2.6653645038604736, + "logits/rejected": -2.6970224380493164, + "logps/chosen": -338.0147399902344, + "logps/rejected": -332.74420166015625, + "loss": 0.6123, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2274622619152069, + "rewards/margins": 0.31383103132247925, + "rewards/rejected": -0.5412932634353638, + "step": 7180 + }, + { + "epoch": 0.94, + "learning_rate": 5.310616921042927e-08, + "logits/chosen": -2.5177793502807617, + "logits/rejected": -2.498081684112549, + "logps/chosen": -274.1446838378906, + "logps/rejected": -251.3614501953125, + "loss": 0.5866, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.16666170954704285, + "rewards/margins": 0.31953534483909607, + "rewards/rejected": -0.4861970841884613, + "step": 7190 + }, + { + "epoch": 0.94, + "learning_rate": 5.079023385830939e-08, + "logits/chosen": -2.4965853691101074, + "logits/rejected": -2.548297643661499, + "logps/chosen": -303.9081115722656, + "logps/rejected": -291.8494873046875, + "loss": 0.5912, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.19232706725597382, + "rewards/margins": 0.3016923666000366, + "rewards/rejected": -0.49401944875717163, + "step": 7200 + }, + { + "epoch": 0.94, + "learning_rate": 4.8525410982695476e-08, + "logits/chosen": -2.664166212081909, + "logits/rejected": -2.532313823699951, + "logps/chosen": -365.9681091308594, + "logps/rejected": -338.4162902832031, + "loss": 0.605, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.3020409643650055, + "rewards/margins": 0.2812599241733551, + "rewards/rejected": -0.5833008885383606, + "step": 7210 + }, + { + "epoch": 0.94, + "learning_rate": 4.6311747848099e-08, + "logits/chosen": -2.6504900455474854, + "logits/rejected": -2.489622116088867, + "logps/chosen": -325.5758056640625, + "logps/rejected": -324.2012634277344, + "loss": 0.5608, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.21951039135456085, + "rewards/margins": 0.33448880910873413, + "rewards/rejected": -0.5539992451667786, + "step": 7220 + }, + { + "epoch": 0.95, + "learning_rate": 4.4149290651382405e-08, + "logits/chosen": -2.489150047302246, + "logits/rejected": -2.5522258281707764, + "logps/chosen": -243.5304412841797, + "logps/rejected": -324.83746337890625, + "loss": 0.5436, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.22564832866191864, + "rewards/margins": 0.46516934037208557, + "rewards/rejected": -0.690817654132843, + "step": 7230 + }, + { + "epoch": 0.95, + "learning_rate": 4.203808452079211e-08, + "logits/chosen": -2.4958624839782715, + "logits/rejected": -2.4828903675079346, + "logps/chosen": -232.5818634033203, + "logps/rejected": -258.32452392578125, + "loss": 0.5691, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.276003897190094, + "rewards/margins": 0.24557428061962128, + "rewards/rejected": -0.5215781927108765, + "step": 7240 + }, + { + "epoch": 0.95, + "learning_rate": 3.9978173515018427e-08, + "logits/chosen": -2.5811498165130615, + "logits/rejected": -2.5918076038360596, + "logps/chosen": -302.5559997558594, + "logps/rejected": -328.662109375, + "loss": 0.6272, + "rewards/accuracies": 0.574999988079071, + "rewards/chosen": -0.3498912751674652, + "rewards/margins": 0.17439976334571838, + "rewards/rejected": -0.5242910385131836, + "step": 7250 + }, + { + "epoch": 0.95, + "learning_rate": 3.7969600622274614e-08, + "logits/chosen": -2.495222806930542, + "logits/rejected": -2.4114508628845215, + "logps/chosen": -374.94720458984375, + "logps/rejected": -313.7791442871094, + "loss": 0.5368, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.24385342001914978, + "rewards/margins": 0.41693535447120667, + "rewards/rejected": -0.6607887148857117, + "step": 7260 + }, + { + "epoch": 0.95, + "learning_rate": 3.601240775940151e-08, + "logits/chosen": -2.627872943878174, + "logits/rejected": -2.585951805114746, + "logps/chosen": -384.087158203125, + "logps/rejected": -376.13232421875, + "loss": 0.601, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.20892560482025146, + "rewards/margins": 0.41255682706832886, + "rewards/rejected": -0.6214824914932251, + "step": 7270 + }, + { + "epoch": 0.95, + "learning_rate": 3.410663577099071e-08, + "logits/chosen": -2.462028980255127, + "logits/rejected": -2.428433418273926, + "logps/chosen": -296.00628662109375, + "logps/rejected": -314.38885498046875, + "loss": 0.5723, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17949989438056946, + "rewards/margins": 0.40048688650131226, + "rewards/rejected": -0.5799866914749146, + "step": 7280 + }, + { + "epoch": 0.95, + "learning_rate": 3.2252324428534986e-08, + "logits/chosen": -2.5399367809295654, + "logits/rejected": -2.384171962738037, + "logps/chosen": -287.9708251953125, + "logps/rejected": -294.95611572265625, + "loss": 0.554, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.1900123953819275, + "rewards/margins": 0.44957780838012695, + "rewards/rejected": -0.6395902037620544, + "step": 7290 + }, + { + "epoch": 0.96, + "learning_rate": 3.0449512429594486e-08, + "logits/chosen": -2.656339406967163, + "logits/rejected": -2.596925973892212, + "logps/chosen": -322.1007995605469, + "logps/rejected": -355.7225646972656, + "loss": 0.5746, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21370860934257507, + "rewards/margins": 0.38849660754203796, + "rewards/rejected": -0.6022052764892578, + "step": 7300 + }, + { + "epoch": 0.96, + "learning_rate": 2.8698237396992956e-08, + "logits/chosen": -2.4962539672851562, + "logits/rejected": -2.4656410217285156, + "logps/chosen": -263.3753662109375, + "logps/rejected": -287.70123291015625, + "loss": 0.5934, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.19082583487033844, + "rewards/margins": 0.47203341126441956, + "rewards/rejected": -0.6628592014312744, + "step": 7310 + }, + { + "epoch": 0.96, + "learning_rate": 2.6998535878030584e-08, + "logits/chosen": -2.484363555908203, + "logits/rejected": -2.4631125926971436, + "logps/chosen": -315.1961669921875, + "logps/rejected": -283.8462829589844, + "loss": 0.5741, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2730049788951874, + "rewards/margins": 0.30888694524765015, + "rewards/rejected": -0.5818918943405151, + "step": 7320 + }, + { + "epoch": 0.96, + "learning_rate": 2.535044334372072e-08, + "logits/chosen": -2.54293155670166, + "logits/rejected": -2.618013620376587, + "logps/chosen": -318.7171325683594, + "logps/rejected": -346.13250732421875, + "loss": 0.5512, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.2942810654640198, + "rewards/margins": 0.4349061846733093, + "rewards/rejected": -0.7291873693466187, + "step": 7330 + }, + { + "epoch": 0.96, + "learning_rate": 2.3753994188051853e-08, + "logits/chosen": -2.343754529953003, + "logits/rejected": -2.3049113750457764, + "logps/chosen": -321.657470703125, + "logps/rejected": -334.9903869628906, + "loss": 0.5791, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2927539646625519, + "rewards/margins": 0.3356403410434723, + "rewards/rejected": -0.6283942461013794, + "step": 7340 + }, + { + "epoch": 0.96, + "learning_rate": 2.220922172726764e-08, + "logits/chosen": -2.6127965450286865, + "logits/rejected": -2.5224764347076416, + "logps/chosen": -301.496337890625, + "logps/rejected": -272.4375305175781, + "loss": 0.6179, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.27615025639533997, + "rewards/margins": 0.1150190606713295, + "rewards/rejected": -0.39116930961608887, + "step": 7350 + }, + { + "epoch": 0.96, + "learning_rate": 2.071615819917244e-08, + "logits/chosen": -2.5463995933532715, + "logits/rejected": -2.495800495147705, + "logps/chosen": -286.66082763671875, + "logps/rejected": -307.84967041015625, + "loss": 0.6042, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.19279077649116516, + "rewards/margins": 0.3158416152000427, + "rewards/rejected": -0.5086323618888855, + "step": 7360 + }, + { + "epoch": 0.96, + "learning_rate": 1.9274834762459393e-08, + "logits/chosen": -2.4749820232391357, + "logits/rejected": -2.576807975769043, + "logps/chosen": -349.1869201660156, + "logps/rejected": -311.72119140625, + "loss": 0.5868, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.31071510910987854, + "rewards/margins": 0.22670575976371765, + "rewards/rejected": -0.5374208688735962, + "step": 7370 + }, + { + "epoch": 0.97, + "learning_rate": 1.7885281496058947e-08, + "logits/chosen": -2.5542681217193604, + "logits/rejected": -2.601315975189209, + "logps/chosen": -292.5190734863281, + "logps/rejected": -291.88873291015625, + "loss": 0.5945, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.09982474148273468, + "rewards/margins": 0.30825820565223694, + "rewards/rejected": -0.40808290243148804, + "step": 7380 + }, + { + "epoch": 0.97, + "learning_rate": 1.654752739851134e-08, + "logits/chosen": -2.478485107421875, + "logits/rejected": -2.5681591033935547, + "logps/chosen": -242.53207397460938, + "logps/rejected": -303.34698486328125, + "loss": 0.5478, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.27765676379203796, + "rewards/margins": 0.29515042901039124, + "rewards/rejected": -0.5728071928024292, + "step": 7390 + }, + { + "epoch": 0.97, + "learning_rate": 1.526160038736235e-08, + "logits/chosen": -2.537532329559326, + "logits/rejected": -2.5868353843688965, + "logps/chosen": -318.7858581542969, + "logps/rejected": -292.2711181640625, + "loss": 0.5937, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.29203277826309204, + "rewards/margins": 0.3736723065376282, + "rewards/rejected": -0.6657050848007202, + "step": 7400 + }, + { + "epoch": 0.97, + "learning_rate": 1.402752729857959e-08, + "logits/chosen": -2.590498685836792, + "logits/rejected": -2.5806334018707275, + "logps/chosen": -261.1465759277344, + "logps/rejected": -281.0189514160156, + "loss": 0.5891, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2757474482059479, + "rewards/margins": 0.19477182626724243, + "rewards/rejected": -0.4705192446708679, + "step": 7410 + }, + { + "epoch": 0.97, + "learning_rate": 1.2845333885992683e-08, + "logits/chosen": -2.4128005504608154, + "logits/rejected": -2.5475845336914062, + "logps/chosen": -273.72332763671875, + "logps/rejected": -293.3398742675781, + "loss": 0.539, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.13520777225494385, + "rewards/margins": 0.4506809115409851, + "rewards/rejected": -0.5858887434005737, + "step": 7420 + }, + { + "epoch": 0.97, + "learning_rate": 1.171504482075675e-08, + "logits/chosen": -2.54884934425354, + "logits/rejected": -2.5620005130767822, + "logps/chosen": -278.064208984375, + "logps/rejected": -289.39794921875, + "loss": 0.5428, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.16827186942100525, + "rewards/margins": 0.4255130887031555, + "rewards/rejected": -0.5937849283218384, + "step": 7430 + }, + { + "epoch": 0.97, + "learning_rate": 1.0636683690836147e-08, + "logits/chosen": -2.526398181915283, + "logits/rejected": -2.4657576084136963, + "logps/chosen": -262.78839111328125, + "logps/rejected": -258.28857421875, + "loss": 0.582, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.17425993084907532, + "rewards/margins": 0.3398754894733429, + "rewards/rejected": -0.5141354203224182, + "step": 7440 + }, + { + "epoch": 0.97, + "learning_rate": 9.610273000513203e-09, + "logits/chosen": -2.614753246307373, + "logits/rejected": -2.573422908782959, + "logps/chosen": -308.4884338378906, + "logps/rejected": -377.4931335449219, + "loss": 0.5838, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.32458263635635376, + "rewards/margins": 0.32800227403640747, + "rewards/rejected": -0.652584969997406, + "step": 7450 + }, + { + "epoch": 0.98, + "learning_rate": 8.635834169918312e-09, + "logits/chosen": -2.557628631591797, + "logits/rejected": -2.5104641914367676, + "logps/chosen": -318.6761474609375, + "logps/rejected": -305.15374755859375, + "loss": 0.6005, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.39393773674964905, + "rewards/margins": 0.14898169040679932, + "rewards/rejected": -0.5429194569587708, + "step": 7460 + }, + { + "epoch": 0.98, + "learning_rate": 7.713387534582506e-09, + "logits/chosen": -2.605743408203125, + "logits/rejected": -2.6812820434570312, + "logps/chosen": -354.9667663574219, + "logps/rejected": -365.39080810546875, + "loss": 0.5956, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3010857403278351, + "rewards/margins": 0.38392147421836853, + "rewards/rejected": -0.6850072145462036, + "step": 7470 + }, + { + "epoch": 0.98, + "learning_rate": 6.84295234501392e-09, + "logits/chosen": -2.485640048980713, + "logits/rejected": -2.4678311347961426, + "logps/chosen": -286.43231201171875, + "logps/rejected": -268.5378723144531, + "loss": 0.5629, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.17162033915519714, + "rewards/margins": 0.5318527817726135, + "rewards/rejected": -0.7034731507301331, + "step": 7480 + }, + { + "epoch": 0.98, + "learning_rate": 6.024546766295325e-09, + "logits/chosen": -2.6134684085845947, + "logits/rejected": -2.539670944213867, + "logps/chosen": -296.3213806152344, + "logps/rejected": -301.2705383300781, + "loss": 0.5466, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.14228995144367218, + "rewards/margins": 0.40069717168807983, + "rewards/rejected": -0.542987048625946, + "step": 7490 + }, + { + "epoch": 0.98, + "learning_rate": 5.2581878777049895e-09, + "logits/chosen": -2.583683490753174, + "logits/rejected": -2.539593458175659, + "logps/chosen": -265.1103820800781, + "logps/rejected": -264.25726318359375, + "loss": 0.5898, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.2404584139585495, + "rewards/margins": 0.2934717535972595, + "rewards/rejected": -0.5339301824569702, + "step": 7500 + }, + { + "epoch": 0.98, + "learning_rate": 4.543891672361411e-09, + "logits/chosen": -2.632657289505005, + "logits/rejected": -2.6300811767578125, + "logps/chosen": -299.6949462890625, + "logps/rejected": -331.4681091308594, + "loss": 0.57, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.18346548080444336, + "rewards/margins": 0.34690719842910767, + "rewards/rejected": -0.5303727388381958, + "step": 7510 + }, + { + "epoch": 0.98, + "learning_rate": 3.881673056887747e-09, + "logits/chosen": -2.5574936866760254, + "logits/rejected": -2.582885265350342, + "logps/chosen": -237.0987091064453, + "logps/rejected": -262.08770751953125, + "loss": 0.5632, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.2373037338256836, + "rewards/margins": 0.37760406732559204, + "rewards/rejected": -0.6149078607559204, + "step": 7520 + }, + { + "epoch": 0.99, + "learning_rate": 3.2715458511023425e-09, + "logits/chosen": -2.514371871948242, + "logits/rejected": -2.5667572021484375, + "logps/chosen": -275.7968444824219, + "logps/rejected": -298.59979248046875, + "loss": 0.5517, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.21118685603141785, + "rewards/margins": 0.3040321171283722, + "rewards/rejected": -0.5152189135551453, + "step": 7530 + }, + { + "epoch": 0.99, + "learning_rate": 2.7135227877289617e-09, + "logits/chosen": -2.454634428024292, + "logits/rejected": -2.5675089359283447, + "logps/chosen": -249.9116668701172, + "logps/rejected": -345.05108642578125, + "loss": 0.6025, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.25211089849472046, + "rewards/margins": 0.18249237537384033, + "rewards/rejected": -0.4346033036708832, + "step": 7540 + }, + { + "epoch": 0.99, + "learning_rate": 2.2076155121328326e-09, + "logits/chosen": -2.563410758972168, + "logits/rejected": -2.6032705307006836, + "logps/chosen": -331.269775390625, + "logps/rejected": -366.534423828125, + "loss": 0.5774, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.24757087230682373, + "rewards/margins": 0.3087003231048584, + "rewards/rejected": -0.5562711954116821, + "step": 7550 + }, + { + "epoch": 0.99, + "learning_rate": 1.7538345820755641e-09, + "logits/chosen": -2.346280574798584, + "logits/rejected": -2.4170234203338623, + "logps/chosen": -263.93865966796875, + "logps/rejected": -321.49615478515625, + "loss": 0.6129, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.2762508988380432, + "rewards/margins": 0.27643775939941406, + "rewards/rejected": -0.5526885986328125, + "step": 7560 + }, + { + "epoch": 0.99, + "learning_rate": 1.3521894674961567e-09, + "logits/chosen": -2.68477201461792, + "logits/rejected": -2.602001667022705, + "logps/chosen": -266.88397216796875, + "logps/rejected": -297.6312255859375, + "loss": 0.5684, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.24672797322273254, + "rewards/margins": 0.292520135641098, + "rewards/rejected": -0.5392481088638306, + "step": 7570 + }, + { + "epoch": 0.99, + "learning_rate": 1.0026885503131023e-09, + "logits/chosen": -2.5589654445648193, + "logits/rejected": -2.558234214782715, + "logps/chosen": -265.538818359375, + "logps/rejected": -298.25372314453125, + "loss": 0.6381, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1422235518693924, + "rewards/margins": 0.3625614047050476, + "rewards/rejected": -0.5047849416732788, + "step": 7580 + }, + { + "epoch": 0.99, + "learning_rate": 7.053391242492491e-10, + "logits/chosen": -2.3764610290527344, + "logits/rejected": -2.380582332611084, + "logps/chosen": -280.76300048828125, + "logps/rejected": -296.69122314453125, + "loss": 0.5852, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.240085169672966, + "rewards/margins": 0.3118717074394226, + "rewards/rejected": -0.551956832408905, + "step": 7590 + }, + { + "epoch": 0.99, + "learning_rate": 4.6014739467997725e-10, + "logits/chosen": -2.493314504623413, + "logits/rejected": -2.4609227180480957, + "logps/chosen": -286.0111083984375, + "logps/rejected": -331.01507568359375, + "loss": 0.6265, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.29493117332458496, + "rewards/margins": 0.3773437440395355, + "rewards/rejected": -0.6722748279571533, + "step": 7600 + }, + { + "epoch": 1.0, + "learning_rate": 2.671184785033032e-10, + "logits/chosen": -2.6753334999084473, + "logits/rejected": -2.719071865081787, + "logps/chosen": -342.070556640625, + "logps/rejected": -344.14990234375, + "loss": 0.6253, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.3149510324001312, + "rewards/margins": 0.22299817204475403, + "rewards/rejected": -0.5379492044448853, + "step": 7610 + }, + { + "epoch": 1.0, + "learning_rate": 1.2625640403302054e-10, + "logits/chosen": -2.422921895980835, + "logits/rejected": -2.439711332321167, + "logps/chosen": -251.980712890625, + "logps/rejected": -311.44805908203125, + "loss": 0.5459, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1920924037694931, + "rewards/margins": 0.4203423857688904, + "rewards/rejected": -0.6124347448348999, + "step": 7620 + }, + { + "epoch": 1.0, + "learning_rate": 3.756411091515588e-11, + "logits/chosen": -2.6092958450317383, + "logits/rejected": -2.5790138244628906, + "logps/chosen": -348.45111083984375, + "logps/rejected": -310.7505798339844, + "loss": 0.5965, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.21164055168628693, + "rewards/margins": 0.3577142059803009, + "rewards/rejected": -0.569354772567749, + "step": 7630 + }, + { + "epoch": 1.0, + "learning_rate": 1.0434500657963143e-12, + "logits/chosen": -2.7101950645446777, + "logits/rejected": -2.600151538848877, + "logps/chosen": -317.154541015625, + "logps/rejected": -331.2071228027344, + "loss": 0.5759, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.22717204689979553, + "rewards/margins": 0.34210166335105896, + "rewards/rejected": -0.5692737102508545, + "step": 7640 + }, + { + "epoch": 1.0, + "step": 7642, + "total_flos": 0.0, + "train_loss": 0.6071469678956156, + "train_runtime": 31823.939, + "train_samples_per_second": 1.921, + "train_steps_per_second": 0.24 + } + ], + "logging_steps": 10, + "max_steps": 7642, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 1000, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}