{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9996297667530544, "eval_steps": 100, "global_step": 1350, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.0, "grad_norm": 0.3203125, "learning_rate": 3.7037037037037036e-09, "logits/chosen": -1.8868012428283691, "logits/rejected": -1.5128765106201172, "logps/chosen": -211.82923889160156, "logps/rejected": -174.22210693359375, "loss": 0.6931, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/margins_max": 0.0, "rewards/margins_min": 0.0, "rewards/margins_std": NaN, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.01, "grad_norm": 0.291015625, "learning_rate": 3.7037037037037036e-08, "logits/chosen": -1.9202638864517212, "logits/rejected": -1.4150642156600952, "logps/chosen": -264.96990966796875, "logps/rejected": -187.2726593017578, "loss": 0.693, "rewards/accuracies": 0.3888888955116272, "rewards/chosen": 0.0007235632510855794, "rewards/margins": -0.0005523893050849438, "rewards/margins_max": -0.0005523893050849438, "rewards/margins_min": -0.0005523893050849438, "rewards/margins_std": NaN, "rewards/rejected": 0.0012759524397552013, "step": 10 }, { "epoch": 0.01, "grad_norm": 0.328125, "learning_rate": 7.407407407407407e-08, "logits/chosen": -2.004002809524536, "logits/rejected": -1.4588892459869385, "logps/chosen": -264.7344970703125, "logps/rejected": -264.04217529296875, "loss": 0.6931, "rewards/accuracies": 0.5, "rewards/chosen": 0.00047433850704692304, "rewards/margins": 8.134842209983617e-05, "rewards/margins_max": 8.134842209983617e-05, "rewards/margins_min": 8.134842209983617e-05, "rewards/margins_std": NaN, "rewards/rejected": 0.0003929900412913412, "step": 20 }, { "epoch": 0.02, "grad_norm": 0.359375, "learning_rate": 1.111111111111111e-07, "logits/chosen": -2.0108284950256348, "logits/rejected": -1.3832088708877563, "logps/chosen": -253.376953125, "logps/rejected": -235.9996337890625, "loss": 0.6931, "rewards/accuracies": 0.550000011920929, "rewards/chosen": 0.0005588417407125235, "rewards/margins": 0.0008742943173274398, "rewards/margins_max": 0.0008742943173274398, "rewards/margins_min": 0.0008742943173274398, "rewards/margins_std": NaN, "rewards/rejected": -0.0003154525766149163, "step": 30 }, { "epoch": 0.03, "grad_norm": 0.310546875, "learning_rate": 1.4814814814814815e-07, "logits/chosen": -1.9831428527832031, "logits/rejected": -1.3419771194458008, "logps/chosen": -249.38821411132812, "logps/rejected": -314.62939453125, "loss": 0.6929, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.0013472252758219838, "rewards/margins": 0.001804034924134612, "rewards/margins_max": 0.001804034924134612, "rewards/margins_min": 0.001804034924134612, "rewards/margins_std": NaN, "rewards/rejected": -0.00045680999755859375, "step": 40 }, { "epoch": 0.04, "grad_norm": 0.314453125, "learning_rate": 1.8518518518518516e-07, "logits/chosen": -2.0799918174743652, "logits/rejected": -1.2689168453216553, "logps/chosen": -269.61492919921875, "logps/rejected": -192.31704711914062, "loss": 0.6929, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.0003735256032086909, "rewards/margins": 0.0009886417537927628, "rewards/margins_max": 0.0009886417537927628, "rewards/margins_min": 0.0009886417537927628, "rewards/margins_std": NaN, "rewards/rejected": -0.000615116092376411, "step": 50 }, { "epoch": 0.04, "grad_norm": 0.310546875, "learning_rate": 2.222222222222222e-07, "logits/chosen": -1.9307972192764282, "logits/rejected": -1.5086009502410889, "logps/chosen": -200.31387329101562, "logps/rejected": -219.31478881835938, "loss": 0.6926, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.00043567089596763253, "rewards/margins": 0.0002629738301038742, "rewards/margins_max": 0.0002629738301038742, "rewards/margins_min": 0.0002629738301038742, "rewards/margins_std": NaN, "rewards/rejected": 0.0001726970513118431, "step": 60 }, { "epoch": 0.05, "grad_norm": 0.32421875, "learning_rate": 2.5925925925925923e-07, "logits/chosen": -2.057626724243164, "logits/rejected": -1.3474286794662476, "logps/chosen": -214.9297637939453, "logps/rejected": -265.07855224609375, "loss": 0.6922, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.00036124425241723657, "rewards/margins": 0.002079057740047574, "rewards/margins_max": 0.002079057740047574, "rewards/margins_min": 0.002079057740047574, "rewards/margins_std": NaN, "rewards/rejected": -0.0017178135458379984, "step": 70 }, { "epoch": 0.06, "grad_norm": 0.33203125, "learning_rate": 2.962962962962963e-07, "logits/chosen": -2.0998992919921875, "logits/rejected": -1.2106634378433228, "logps/chosen": -266.18182373046875, "logps/rejected": -211.5884552001953, "loss": 0.6926, "rewards/accuracies": 0.75, "rewards/chosen": 0.0019807242788374424, "rewards/margins": 0.003271686378866434, "rewards/margins_max": 0.003271686378866434, "rewards/margins_min": 0.003271686378866434, "rewards/margins_std": NaN, "rewards/rejected": -0.0012909623328596354, "step": 80 }, { "epoch": 0.07, "grad_norm": 0.337890625, "learning_rate": 3.333333333333333e-07, "logits/chosen": -2.0147571563720703, "logits/rejected": -1.2995073795318604, "logps/chosen": -247.42263793945312, "logps/rejected": -228.5316619873047, "loss": 0.6919, "rewards/accuracies": 0.5, "rewards/chosen": 0.001941970898769796, "rewards/margins": 0.0006094207055866718, "rewards/margins_max": 0.0006094207055866718, "rewards/margins_min": 0.0006094207055866718, "rewards/margins_std": NaN, "rewards/rejected": 0.0013325500767678022, "step": 90 }, { "epoch": 0.07, "grad_norm": 0.322265625, "learning_rate": 3.703703703703703e-07, "logits/chosen": -1.983267068862915, "logits/rejected": -1.3546688556671143, "logps/chosen": -202.480224609375, "logps/rejected": -174.5247344970703, "loss": 0.6916, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.0025401152670383453, "rewards/margins": 0.002821914618834853, "rewards/margins_max": 0.002821914618834853, "rewards/margins_min": 0.002821914618834853, "rewards/margins_std": NaN, "rewards/rejected": -0.0002817992935888469, "step": 100 }, { "epoch": 0.08, "grad_norm": 0.3125, "learning_rate": 4.0740740740740737e-07, "logits/chosen": -2.0803046226501465, "logits/rejected": -1.4716895818710327, "logps/chosen": -225.15560913085938, "logps/rejected": -256.0934753417969, "loss": 0.6912, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.003935464657843113, "rewards/margins": 0.0047843605279922485, "rewards/margins_max": 0.0047843605279922485, "rewards/margins_min": 0.0047843605279922485, "rewards/margins_std": NaN, "rewards/rejected": -0.0008488958701491356, "step": 110 }, { "epoch": 0.09, "grad_norm": 0.298828125, "learning_rate": 4.444444444444444e-07, "logits/chosen": -1.8733142614364624, "logits/rejected": -1.7225223779678345, "logps/chosen": -172.91468811035156, "logps/rejected": -199.7720489501953, "loss": 0.6909, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.00166155060287565, "rewards/margins": 0.0015941675519570708, "rewards/margins_max": 0.0015941675519570708, "rewards/margins_min": 0.0015941675519570708, "rewards/margins_std": NaN, "rewards/rejected": 6.73827962600626e-05, "step": 120 }, { "epoch": 0.1, "grad_norm": 0.326171875, "learning_rate": 4.814814814814814e-07, "logits/chosen": -1.9976625442504883, "logits/rejected": -1.2138649225234985, "logps/chosen": -239.32485961914062, "logps/rejected": -240.15597534179688, "loss": 0.6899, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.0042144907638430595, "rewards/margins": 0.005392442457377911, "rewards/margins_max": 0.005392442457377911, "rewards/margins_min": 0.005392442457377911, "rewards/margins_std": NaN, "rewards/rejected": -0.001177951809950173, "step": 130 }, { "epoch": 0.1, "grad_norm": 0.291015625, "learning_rate": 4.999791074638593e-07, "logits/chosen": -2.195509433746338, "logits/rejected": -1.5012781620025635, "logps/chosen": -216.47579956054688, "logps/rejected": -207.82186889648438, "loss": 0.6904, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.005214618518948555, "rewards/margins": 0.006003814749419689, "rewards/margins_max": 0.006003814749419689, "rewards/margins_min": 0.006003814749419689, "rewards/margins_std": NaN, "rewards/rejected": -0.0007891958812251687, "step": 140 }, { "epoch": 0.11, "grad_norm": 0.30078125, "learning_rate": 4.998119881260575e-07, "logits/chosen": -2.0202810764312744, "logits/rejected": -1.4483054876327515, "logps/chosen": -212.3457489013672, "logps/rejected": -211.62197875976562, "loss": 0.689, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.004780811257660389, "rewards/margins": 0.008003057911992073, "rewards/margins_max": 0.008003057911992073, "rewards/margins_min": 0.008003057911992073, "rewards/margins_std": NaN, "rewards/rejected": -0.003222248051315546, "step": 150 }, { "epoch": 0.12, "grad_norm": 0.30859375, "learning_rate": 4.99477861175283e-07, "logits/chosen": -2.0366463661193848, "logits/rejected": -1.3878607749938965, "logps/chosen": -231.7950439453125, "logps/rejected": -239.493896484375, "loss": 0.6886, "rewards/accuracies": 1.0, "rewards/chosen": 0.007523299660533667, "rewards/margins": 0.0094174575060606, "rewards/margins_max": 0.0094174575060606, "rewards/margins_min": 0.0094174575060606, "rewards/margins_std": NaN, "rewards/rejected": -0.0018941573798656464, "step": 160 }, { "epoch": 0.13, "grad_norm": 0.30078125, "learning_rate": 4.989769499865024e-07, "logits/chosen": -1.8927358388900757, "logits/rejected": -1.4275254011154175, "logps/chosen": -239.3990936279297, "logps/rejected": -282.657958984375, "loss": 0.6885, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.009550971910357475, "rewards/margins": 0.011694083921611309, "rewards/margins_max": 0.011694083921611309, "rewards/margins_min": 0.011694083921611309, "rewards/margins_std": NaN, "rewards/rejected": -0.0021431122440844774, "step": 170 }, { "epoch": 0.13, "grad_norm": 0.32421875, "learning_rate": 4.983095894354857e-07, "logits/chosen": -2.0323657989501953, "logits/rejected": -1.417519211769104, "logps/chosen": -321.04107666015625, "logps/rejected": -304.40753173828125, "loss": 0.6874, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.006479477044194937, "rewards/margins": 0.010976837947964668, "rewards/margins_max": 0.010976837947964668, "rewards/margins_min": 0.010976837947964668, "rewards/margins_std": NaN, "rewards/rejected": -0.00449735950678587, "step": 180 }, { "epoch": 0.14, "grad_norm": 0.3046875, "learning_rate": 4.974762256749318e-07, "logits/chosen": -2.036602020263672, "logits/rejected": -1.5517603158950806, "logps/chosen": -231.96826171875, "logps/rejected": -237.5748291015625, "loss": 0.6869, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.005657371133565903, "rewards/margins": 0.008646010421216488, "rewards/margins_max": 0.008646010421216488, "rewards/margins_min": 0.008646010421216488, "rewards/margins_std": NaN, "rewards/rejected": -0.0029886397533118725, "step": 190 }, { "epoch": 0.15, "grad_norm": 0.326171875, "learning_rate": 4.96477415836199e-07, "logits/chosen": -1.938863754272461, "logits/rejected": -1.2694227695465088, "logps/chosen": -244.3894805908203, "logps/rejected": -208.3003387451172, "loss": 0.6864, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.005547370761632919, "rewards/margins": 0.012681066989898682, "rewards/margins_max": 0.012681066989898682, "rewards/margins_min": 0.012681066989898682, "rewards/margins_std": NaN, "rewards/rejected": -0.007133697625249624, "step": 200 }, { "epoch": 0.16, "grad_norm": 0.267578125, "learning_rate": 4.953138276568461e-07, "logits/chosen": -1.997301697731018, "logits/rejected": -1.3399479389190674, "logps/chosen": -301.72332763671875, "logps/rejected": -227.8885955810547, "loss": 0.6853, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.008855095133185387, "rewards/margins": 0.019029559567570686, "rewards/margins_max": 0.019029559567570686, "rewards/margins_min": 0.019029559567570686, "rewards/margins_std": NaN, "rewards/rejected": -0.010174466297030449, "step": 210 }, { "epoch": 0.16, "grad_norm": 0.353515625, "learning_rate": 4.939862390342259e-07, "logits/chosen": -1.9495147466659546, "logits/rejected": -1.5158745050430298, "logps/chosen": -216.283935546875, "logps/rejected": -199.36839294433594, "loss": 0.6843, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.00766110559925437, "rewards/margins": 0.01607651077210903, "rewards/margins_max": 0.01607651077210903, "rewards/margins_min": 0.01607651077210903, "rewards/margins_std": NaN, "rewards/rejected": -0.00841540563851595, "step": 220 }, { "epoch": 0.17, "grad_norm": 0.29296875, "learning_rate": 4.924955375054358e-07, "logits/chosen": -2.16361665725708, "logits/rejected": -1.274595022201538, "logps/chosen": -189.79823303222656, "logps/rejected": -198.684814453125, "loss": 0.6845, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.012809082865715027, "rewards/margins": 0.015618041157722473, "rewards/margins_max": 0.015618041157722473, "rewards/margins_min": 0.015618041157722473, "rewards/margins_std": NaN, "rewards/rejected": -0.002808959688991308, "step": 230 }, { "epoch": 0.18, "grad_norm": 0.337890625, "learning_rate": 4.908427196539701e-07, "logits/chosen": -2.0759100914001465, "logits/rejected": -1.4362943172454834, "logps/chosen": -224.01724243164062, "logps/rejected": -210.9996337890625, "loss": 0.6831, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.011357143521308899, "rewards/margins": 0.015575545839965343, "rewards/margins_max": 0.015575545839965343, "rewards/margins_min": 0.015575545839965343, "rewards/margins_std": NaN, "rewards/rejected": -0.0042184023186564445, "step": 240 }, { "epoch": 0.19, "grad_norm": 0.337890625, "learning_rate": 4.890288904434699e-07, "logits/chosen": -2.0894532203674316, "logits/rejected": -1.2666500806808472, "logps/chosen": -258.41571044921875, "logps/rejected": -249.0290985107422, "loss": 0.6807, "rewards/accuracies": 1.0, "rewards/chosen": 0.026370421051979065, "rewards/margins": 0.03443429246544838, "rewards/margins_max": 0.03443429246544838, "rewards/margins_min": 0.03443429246544838, "rewards/margins_std": NaN, "rewards/rejected": -0.008063869550824165, "step": 250 }, { "epoch": 0.19, "grad_norm": 0.310546875, "learning_rate": 4.87055262479019e-07, "logits/chosen": -2.0931239128112793, "logits/rejected": -1.5411086082458496, "logps/chosen": -231.93203735351562, "logps/rejected": -209.02432250976562, "loss": 0.6831, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.012660378590226173, "rewards/margins": 0.020816108211874962, "rewards/margins_max": 0.020816108211874962, "rewards/margins_min": 0.020816108211874962, "rewards/margins_std": NaN, "rewards/rejected": -0.008155731484293938, "step": 260 }, { "epoch": 0.2, "grad_norm": 0.337890625, "learning_rate": 4.849231551964771e-07, "logits/chosen": -2.129526138305664, "logits/rejected": -1.3961608409881592, "logps/chosen": -257.1175231933594, "logps/rejected": -238.4982147216797, "loss": 0.6812, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.022793341428041458, "rewards/margins": 0.02573349140584469, "rewards/margins_max": 0.02573349140584469, "rewards/margins_min": 0.02573349140584469, "rewards/margins_std": NaN, "rewards/rejected": -0.00294014741666615, "step": 270 }, { "epoch": 0.21, "grad_norm": 0.34765625, "learning_rate": 4.826339939803922e-07, "logits/chosen": -2.051805257797241, "logits/rejected": -1.4157917499542236, "logps/chosen": -226.42919921875, "logps/rejected": -268.73980712890625, "loss": 0.6803, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.016779452562332153, "rewards/margins": 0.027814749628305435, "rewards/margins_max": 0.027814749628305435, "rewards/margins_min": 0.027814749628305435, "rewards/margins_std": NaN, "rewards/rejected": -0.011035297997295856, "step": 280 }, { "epoch": 0.21, "grad_norm": 0.2890625, "learning_rate": 4.801893092110846e-07, "logits/chosen": -2.0515878200531006, "logits/rejected": -1.403578519821167, "logps/chosen": -200.06935119628906, "logps/rejected": -222.7530059814453, "loss": 0.6798, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.02002173662185669, "rewards/margins": 0.03395923227071762, "rewards/margins_max": 0.03395923227071762, "rewards/margins_min": 0.03395923227071762, "rewards/margins_std": NaN, "rewards/rejected": -0.013937492854893208, "step": 290 }, { "epoch": 0.22, "grad_norm": 0.34375, "learning_rate": 4.775907352415367e-07, "logits/chosen": -2.056366443634033, "logits/rejected": -1.4939775466918945, "logps/chosen": -227.35498046875, "logps/rejected": -234.95849609375, "loss": 0.6792, "rewards/accuracies": 1.0, "rewards/chosen": 0.017659524455666542, "rewards/margins": 0.028245586901903152, "rewards/margins_max": 0.028245586901903152, "rewards/margins_min": 0.028245586901903152, "rewards/margins_std": NaN, "rewards/rejected": -0.01058606244623661, "step": 300 }, { "epoch": 0.23, "grad_norm": 0.359375, "learning_rate": 4.7484000930477327e-07, "logits/chosen": -2.1390914916992188, "logits/rejected": -1.4254201650619507, "logps/chosen": -258.49652099609375, "logps/rejected": -261.0328063964844, "loss": 0.6774, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.02551867440342903, "rewards/margins": 0.03578513488173485, "rewards/margins_max": 0.03578513488173485, "rewards/margins_min": 0.03578513488173485, "rewards/margins_std": NaN, "rewards/rejected": -0.010266458615660667, "step": 310 }, { "epoch": 0.24, "grad_norm": 0.359375, "learning_rate": 4.71938970352464e-07, "logits/chosen": -2.0027215480804443, "logits/rejected": -1.6378333568572998, "logps/chosen": -177.40463256835938, "logps/rejected": -213.1610107421875, "loss": 0.6789, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.01185548771172762, "rewards/margins": 0.021204955875873566, "rewards/margins_max": 0.021204955875873566, "rewards/margins_min": 0.021204955875873566, "rewards/margins_std": NaN, "rewards/rejected": -0.009349468164145947, "step": 320 }, { "epoch": 0.24, "grad_norm": 0.349609375, "learning_rate": 4.688895578255227e-07, "logits/chosen": -2.0879385471343994, "logits/rejected": -1.248313546180725, "logps/chosen": -262.8072509765625, "logps/rejected": -248.21780395507812, "loss": 0.6765, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.01555053610354662, "rewards/margins": 0.032046493142843246, "rewards/margins_max": 0.032046493142843246, "rewards/margins_min": 0.032046493142843246, "rewards/margins_std": NaN, "rewards/rejected": -0.016495956107974052, "step": 330 }, { "epoch": 0.25, "grad_norm": 0.34765625, "learning_rate": 4.656938103575271e-07, "logits/chosen": -2.172673463821411, "logits/rejected": -1.4102470874786377, "logps/chosen": -270.31378173828125, "logps/rejected": -231.1504364013672, "loss": 0.6758, "rewards/accuracies": 1.0, "rewards/chosen": 0.023389603942632675, "rewards/margins": 0.043514709919691086, "rewards/margins_max": 0.043514709919691086, "rewards/margins_min": 0.043514709919691086, "rewards/margins_std": NaN, "rewards/rejected": -0.02012510970234871, "step": 340 }, { "epoch": 0.26, "grad_norm": 0.28125, "learning_rate": 4.6235386441182434e-07, "logits/chosen": -2.126311779022217, "logits/rejected": -1.5584805011749268, "logps/chosen": -217.91452026367188, "logps/rejected": -242.47653198242188, "loss": 0.6768, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.02275378443300724, "rewards/margins": 0.031895603984594345, "rewards/margins_max": 0.031895603984594345, "rewards/margins_min": 0.031895603984594345, "rewards/margins_std": NaN, "rewards/rejected": -0.009141822345554829, "step": 350 }, { "epoch": 0.27, "grad_norm": 0.33203125, "learning_rate": 4.588719528532341e-07, "logits/chosen": -2.138247013092041, "logits/rejected": -1.4281116724014282, "logps/chosen": -227.18411254882812, "logps/rejected": -220.9769287109375, "loss": 0.6768, "rewards/accuracies": 1.0, "rewards/chosen": 0.020104510709643364, "rewards/margins": 0.04013504832983017, "rewards/margins_max": 0.04013504832983017, "rewards/margins_min": 0.04013504832983017, "rewards/margins_std": NaN, "rewards/rejected": -0.020030539482831955, "step": 360 }, { "epoch": 0.27, "grad_norm": 0.349609375, "learning_rate": 4.5525040345530465e-07, "logits/chosen": -2.065033435821533, "logits/rejected": -1.3876053094863892, "logps/chosen": -255.1400604248047, "logps/rejected": -341.7924499511719, "loss": 0.6745, "rewards/accuracies": 1.0, "rewards/chosen": 0.030053580179810524, "rewards/margins": 0.04531164467334747, "rewards/margins_max": 0.04531164467334747, "rewards/margins_min": 0.04531164467334747, "rewards/margins_std": NaN, "rewards/rejected": -0.015258069150149822, "step": 370 }, { "epoch": 0.28, "grad_norm": 0.357421875, "learning_rate": 4.5149163734411855e-07, "logits/chosen": -1.9663784503936768, "logits/rejected": -1.4839365482330322, "logps/chosen": -257.76336669921875, "logps/rejected": -252.73291015625, "loss": 0.6729, "rewards/accuracies": 1.0, "rewards/chosen": 0.01958070509135723, "rewards/margins": 0.04934977740049362, "rewards/margins_max": 0.04934977740049362, "rewards/margins_min": 0.04934977740049362, "rewards/margins_std": NaN, "rewards/rejected": -0.02976907417178154, "step": 380 }, { "epoch": 0.29, "grad_norm": 0.314453125, "learning_rate": 4.475981673796898e-07, "logits/chosen": -2.087552309036255, "logits/rejected": -1.1439874172210693, "logps/chosen": -238.96328735351562, "logps/rejected": -177.21859741210938, "loss": 0.6734, "rewards/accuracies": 1.0, "rewards/chosen": 0.028685113415122032, "rewards/margins": 0.04815841466188431, "rewards/margins_max": 0.04815841466188431, "rewards/margins_min": 0.04815841466188431, "rewards/margins_std": NaN, "rewards/rejected": -0.019473304972052574, "step": 390 }, { "epoch": 0.3, "grad_norm": 0.33203125, "learning_rate": 4.435725964760331e-07, "logits/chosen": -2.0537381172180176, "logits/rejected": -1.5530219078063965, "logps/chosen": -198.19747924804688, "logps/rejected": -225.1847686767578, "loss": 0.6738, "rewards/accuracies": 1.0, "rewards/chosen": 0.025270873680710793, "rewards/margins": 0.04031512513756752, "rewards/margins_max": 0.04031512513756752, "rewards/margins_min": 0.04031512513756752, "rewards/margins_std": NaN, "rewards/rejected": -0.01504424773156643, "step": 400 }, { "epoch": 0.3, "grad_norm": 0.337890625, "learning_rate": 4.3941761586102946e-07, "logits/chosen": -2.0906436443328857, "logits/rejected": -1.287006139755249, "logps/chosen": -254.64700317382812, "logps/rejected": -214.3684539794922, "loss": 0.6727, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.029230216518044472, "rewards/margins": 0.04698926955461502, "rewards/margins_max": 0.04698926955461502, "rewards/margins_min": 0.04698926955461502, "rewards/margins_std": NaN, "rewards/rejected": -0.0177590511739254, "step": 410 }, { "epoch": 0.31, "grad_norm": 0.294921875, "learning_rate": 4.3513600327725117e-07, "logits/chosen": -1.9869972467422485, "logits/rejected": -1.4108494520187378, "logps/chosen": -261.79034423828125, "logps/rejected": -244.15512084960938, "loss": 0.6719, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.02686556614935398, "rewards/margins": 0.04367043077945709, "rewards/margins_max": 0.04367043077945709, "rewards/margins_min": 0.04367043077945709, "rewards/margins_std": NaN, "rewards/rejected": -0.01680486463010311, "step": 420 }, { "epoch": 0.32, "grad_norm": 0.361328125, "learning_rate": 4.307306211249482e-07, "logits/chosen": -1.9715445041656494, "logits/rejected": -1.4791746139526367, "logps/chosen": -181.6294403076172, "logps/rejected": -219.4192352294922, "loss": 0.6697, "rewards/accuracies": 1.0, "rewards/chosen": 0.02723332680761814, "rewards/margins": 0.05029752105474472, "rewards/margins_max": 0.05029752105474472, "rewards/margins_min": 0.05029752105474472, "rewards/margins_std": NaN, "rewards/rejected": -0.023064199835062027, "step": 430 }, { "epoch": 0.33, "grad_norm": 0.322265625, "learning_rate": 4.26204414548439e-07, "logits/chosen": -2.0978024005889893, "logits/rejected": -1.3412537574768066, "logps/chosen": -257.7082824707031, "logps/rejected": -223.86495971679688, "loss": 0.6714, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.025436753407120705, "rewards/margins": 0.050586361438035965, "rewards/margins_max": 0.050586361438035965, "rewards/margins_min": 0.050586361438035965, "rewards/margins_std": NaN, "rewards/rejected": -0.025149613618850708, "step": 440 }, { "epoch": 0.33, "grad_norm": 0.3046875, "learning_rate": 4.2156040946718343e-07, "logits/chosen": -2.0477805137634277, "logits/rejected": -1.5934501886367798, "logps/chosen": -228.3242950439453, "logps/rejected": -272.94488525390625, "loss": 0.6667, "rewards/accuracies": 1.0, "rewards/chosen": 0.03539489209651947, "rewards/margins": 0.05500010401010513, "rewards/margins_max": 0.05500010401010513, "rewards/margins_min": 0.05500010401010513, "rewards/margins_std": NaN, "rewards/rejected": -0.019605208188295364, "step": 450 }, { "epoch": 0.34, "grad_norm": 0.30859375, "learning_rate": 4.1680171055285484e-07, "logits/chosen": -2.0926973819732666, "logits/rejected": -1.2051217555999756, "logps/chosen": -291.447021484375, "logps/rejected": -230.220947265625, "loss": 0.6697, "rewards/accuracies": 1.0, "rewards/chosen": 0.027433469891548157, "rewards/margins": 0.04781699925661087, "rewards/margins_max": 0.04781699925661087, "rewards/margins_min": 0.04781699925661087, "rewards/margins_std": NaN, "rewards/rejected": -0.020383525639772415, "step": 460 }, { "epoch": 0.35, "grad_norm": 0.3203125, "learning_rate": 4.119314991537647e-07, "logits/chosen": -2.0807855129241943, "logits/rejected": -1.3683123588562012, "logps/chosen": -234.7393035888672, "logps/rejected": -259.2615051269531, "loss": 0.6693, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.03391563892364502, "rewards/margins": 0.06610719114542007, "rewards/margins_max": 0.06610719114542007, "rewards/margins_min": 0.06610719114542007, "rewards/margins_std": NaN, "rewards/rejected": -0.032191552221775055, "step": 470 }, { "epoch": 0.36, "grad_norm": 0.328125, "learning_rate": 4.0695303116802467e-07, "logits/chosen": -1.9418798685073853, "logits/rejected": -1.5194308757781982, "logps/chosen": -239.70016479492188, "logps/rejected": -230.9000701904297, "loss": 0.6673, "rewards/accuracies": 1.0, "rewards/chosen": 0.029919007793068886, "rewards/margins": 0.05285835266113281, "rewards/margins_max": 0.05285835266113281, "rewards/margins_min": 0.05285835266113281, "rewards/margins_std": NaN, "rewards/rejected": -0.022939343005418777, "step": 480 }, { "epoch": 0.36, "grad_norm": 0.33203125, "learning_rate": 4.0186963486687086e-07, "logits/chosen": -2.1395695209503174, "logits/rejected": -1.520246148109436, "logps/chosen": -181.22784423828125, "logps/rejected": -206.8192901611328, "loss": 0.6682, "rewards/accuracies": 1.0, "rewards/chosen": 0.03531926870346069, "rewards/margins": 0.05207179859280586, "rewards/margins_max": 0.05207179859280586, "rewards/margins_min": 0.05207179859280586, "rewards/margins_std": NaN, "rewards/rejected": -0.01675252802670002, "step": 490 }, { "epoch": 0.37, "grad_norm": 0.349609375, "learning_rate": 3.966847086696045e-07, "logits/chosen": -2.017302989959717, "logits/rejected": -1.3829624652862549, "logps/chosen": -231.52651977539062, "logps/rejected": -221.0729217529297, "loss": 0.6692, "rewards/accuracies": 1.0, "rewards/chosen": 0.023424681276082993, "rewards/margins": 0.05106498673558235, "rewards/margins_max": 0.05106498673558235, "rewards/margins_min": 0.05106498673558235, "rewards/margins_std": NaN, "rewards/rejected": -0.02764030732214451, "step": 500 }, { "epoch": 0.38, "grad_norm": 0.271484375, "learning_rate": 3.9140171887163466e-07, "logits/chosen": -1.9985767602920532, "logits/rejected": -1.6083755493164062, "logps/chosen": -280.05303955078125, "logps/rejected": -237.509521484375, "loss": 0.6681, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.016823355108499527, "rewards/margins": 0.0439620167016983, "rewards/margins_max": 0.0439620167016983, "rewards/margins_min": 0.0439620167016983, "rewards/margins_std": NaN, "rewards/rejected": -0.027138665318489075, "step": 510 }, { "epoch": 0.39, "grad_norm": 0.283203125, "learning_rate": 3.8602419732714566e-07, "logits/chosen": -2.036795139312744, "logits/rejected": -1.471296787261963, "logps/chosen": -256.37481689453125, "logps/rejected": -199.65538024902344, "loss": 0.6697, "rewards/accuracies": 1.0, "rewards/chosen": 0.026260962709784508, "rewards/margins": 0.038443367928266525, "rewards/margins_max": 0.038443367928266525, "rewards/margins_min": 0.038443367928266525, "rewards/margins_std": NaN, "rewards/rejected": -0.012182402424514294, "step": 520 }, { "epoch": 0.39, "grad_norm": 0.322265625, "learning_rate": 3.805557390879346e-07, "logits/chosen": -1.9529104232788086, "logits/rejected": -1.5426490306854248, "logps/chosen": -182.79574584960938, "logps/rejected": -194.38156127929688, "loss": 0.6679, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.020470619201660156, "rewards/margins": 0.049423329532146454, "rewards/margins_max": 0.049423329532146454, "rewards/margins_min": 0.049423329532146454, "rewards/margins_std": NaN, "rewards/rejected": -0.02895270846784115, "step": 530 }, { "epoch": 0.4, "grad_norm": 0.314453125, "learning_rate": 3.75e-07, "logits/chosen": -2.0001742839813232, "logits/rejected": -1.2963764667510986, "logps/chosen": -230.01394653320312, "logps/rejected": -232.5394287109375, "loss": 0.6667, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.037200070917606354, "rewards/margins": 0.0586659200489521, "rewards/margins_max": 0.0586659200489521, "rewards/margins_min": 0.0586659200489521, "rewards/margins_std": NaN, "rewards/rejected": -0.0214658472687006, "step": 540 }, { "epoch": 0.41, "grad_norm": 0.318359375, "learning_rate": 3.693606942594872e-07, "logits/chosen": -2.0440757274627686, "logits/rejected": -1.3815096616744995, "logps/chosen": -217.4425811767578, "logps/rejected": -171.90591430664062, "loss": 0.6676, "rewards/accuracies": 1.0, "rewards/chosen": 0.02869860827922821, "rewards/margins": 0.0414133220911026, "rewards/margins_max": 0.0414133220911026, "rewards/margins_min": 0.0414133220911026, "rewards/margins_std": NaN, "rewards/rejected": -0.012714711017906666, "step": 550 }, { "epoch": 0.41, "grad_norm": 0.345703125, "learning_rate": 3.636415919296251e-07, "logits/chosen": -1.9253566265106201, "logits/rejected": -1.6695436239242554, "logps/chosen": -171.96823120117188, "logps/rejected": -167.778076171875, "loss": 0.667, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.02604065276682377, "rewards/margins": 0.04568729177117348, "rewards/margins_max": 0.04568729177117348, "rewards/margins_min": 0.04568729177117348, "rewards/margins_std": NaN, "rewards/rejected": -0.01964663527905941, "step": 560 }, { "epoch": 0.42, "grad_norm": 0.3359375, "learning_rate": 3.5784651642031337e-07, "logits/chosen": -1.8566385507583618, "logits/rejected": -1.6557972431182861, "logps/chosen": -224.4451904296875, "logps/rejected": -270.01776123046875, "loss": 0.6701, "rewards/accuracies": 1.0, "rewards/chosen": 0.004797924309968948, "rewards/margins": 0.041053663939237595, "rewards/margins_max": 0.041053663939237595, "rewards/margins_min": 0.041053663939237595, "rewards/margins_std": NaN, "rewards/rejected": -0.036255739629268646, "step": 570 }, { "epoch": 0.43, "grad_norm": 0.353515625, "learning_rate": 3.519793419320459e-07, "logits/chosen": -1.9248411655426025, "logits/rejected": -1.1966019868850708, "logps/chosen": -284.9685974121094, "logps/rejected": -223.8994140625, "loss": 0.6664, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.024664491415023804, "rewards/margins": 0.05492042750120163, "rewards/margins_max": 0.05492042750120163, "rewards/margins_min": 0.05492042750120163, "rewards/margins_std": NaN, "rewards/rejected": -0.030255932360887527, "step": 580 }, { "epoch": 0.44, "grad_norm": 0.333984375, "learning_rate": 3.460439908658794e-07, "logits/chosen": -2.007855176925659, "logits/rejected": -1.1754387617111206, "logps/chosen": -349.209228515625, "logps/rejected": -206.0282440185547, "loss": 0.6687, "rewards/accuracies": 1.0, "rewards/chosen": 0.0192419346421957, "rewards/margins": 0.04281260445713997, "rewards/margins_max": 0.04281260445713997, "rewards/margins_min": 0.04281260445713997, "rewards/margins_std": NaN, "rewards/rejected": -0.023570667952299118, "step": 590 }, { "epoch": 0.44, "grad_norm": 0.34765625, "learning_rate": 3.400444312011776e-07, "logits/chosen": -2.059713840484619, "logits/rejected": -1.321000576019287, "logps/chosen": -265.93463134765625, "logps/rejected": -285.9882507324219, "loss": 0.6689, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.0264985803514719, "rewards/margins": 0.05372191593050957, "rewards/margins_max": 0.05372191593050957, "rewards/margins_min": 0.05372191593050957, "rewards/margins_std": NaN, "rewards/rejected": -0.027223343029618263, "step": 600 }, { "epoch": 0.45, "grad_norm": 0.326171875, "learning_rate": 3.3398467384288554e-07, "logits/chosen": -2.0164129734039307, "logits/rejected": -1.428724765777588, "logps/chosen": -174.00619506835938, "logps/rejected": -235.1026153564453, "loss": 0.665, "rewards/accuracies": 1.0, "rewards/chosen": 0.018517404794692993, "rewards/margins": 0.06648163497447968, "rewards/margins_max": 0.06648163497447968, "rewards/margins_min": 0.06648163497447968, "rewards/margins_std": NaN, "rewards/rejected": -0.047964222729206085, "step": 610 }, { "epoch": 0.46, "grad_norm": 0.37890625, "learning_rate": 3.2786876994010616e-07, "logits/chosen": -1.9753456115722656, "logits/rejected": -1.5021337270736694, "logps/chosen": -207.2568359375, "logps/rejected": -196.98239135742188, "loss": 0.6654, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.026976531371474266, "rewards/margins": 0.05444413423538208, "rewards/margins_max": 0.05444413423538208, "rewards/margins_min": 0.05444413423538208, "rewards/margins_std": NaN, "rewards/rejected": -0.027467604726552963, "step": 620 }, { "epoch": 0.47, "grad_norm": 0.318359375, "learning_rate": 3.2170080817777257e-07, "logits/chosen": -2.0977303981781006, "logits/rejected": -1.3275058269500732, "logps/chosen": -280.2716979980469, "logps/rejected": -243.47763061523438, "loss": 0.6658, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.0296818558126688, "rewards/margins": 0.054945122450590134, "rewards/margins_max": 0.054945122450590134, "rewards/margins_min": 0.054945122450590134, "rewards/margins_std": NaN, "rewards/rejected": -0.025263268500566483, "step": 630 }, { "epoch": 0.47, "grad_norm": 0.349609375, "learning_rate": 3.1548491204322647e-07, "logits/chosen": -2.0041770935058594, "logits/rejected": -1.5506256818771362, "logps/chosen": -211.4202423095703, "logps/rejected": -374.6882629394531, "loss": 0.6674, "rewards/accuracies": 1.0, "rewards/chosen": 0.022962290793657303, "rewards/margins": 0.05601681396365166, "rewards/margins_max": 0.05601681396365166, "rewards/margins_min": 0.05601681396365166, "rewards/margins_std": NaN, "rewards/rejected": -0.033054523169994354, "step": 640 }, { "epoch": 0.48, "grad_norm": 0.35546875, "learning_rate": 3.0922523706952976e-07, "logits/chosen": -2.059875249862671, "logits/rejected": -1.407546877861023, "logps/chosen": -208.85415649414062, "logps/rejected": -180.44479370117188, "loss": 0.667, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.03480786085128784, "rewards/margins": 0.05192246288061142, "rewards/margins_max": 0.05192246288061142, "rewards/margins_min": 0.05192246288061142, "rewards/margins_std": NaN, "rewards/rejected": -0.017114607617259026, "step": 650 }, { "epoch": 0.49, "grad_norm": 0.359375, "learning_rate": 3.029259680573527e-07, "logits/chosen": -2.1605982780456543, "logits/rejected": -1.3930118083953857, "logps/chosen": -225.4413299560547, "logps/rejected": -222.64828491210938, "loss": 0.6667, "rewards/accuracies": 1.0, "rewards/chosen": 0.02937215566635132, "rewards/margins": 0.053660131990909576, "rewards/margins_max": 0.053660131990909576, "rewards/margins_min": 0.053660131990909576, "rewards/margins_std": NaN, "rewards/rejected": -0.02428797259926796, "step": 660 }, { "epoch": 0.5, "grad_norm": 0.28515625, "learning_rate": 2.9659131627729555e-07, "logits/chosen": -2.1441378593444824, "logits/rejected": -1.2556051015853882, "logps/chosen": -273.76507568359375, "logps/rejected": -173.42489624023438, "loss": 0.6632, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.039141856133937836, "rewards/margins": 0.05589345097541809, "rewards/margins_max": 0.05589345097541809, "rewards/margins_min": 0.05589345097541809, "rewards/margins_std": NaN, "rewards/rejected": -0.016751592978835106, "step": 670 }, { "epoch": 0.5, "grad_norm": 0.3515625, "learning_rate": 2.9022551665451433e-07, "logits/chosen": -2.107771158218384, "logits/rejected": -1.0939693450927734, "logps/chosen": -241.0987091064453, "logps/rejected": -229.3813018798828, "loss": 0.6646, "rewards/accuracies": 1.0, "rewards/chosen": 0.043823134154081345, "rewards/margins": 0.08219417929649353, "rewards/margins_max": 0.08219417929649353, "rewards/margins_min": 0.08219417929649353, "rewards/margins_std": NaN, "rewards/rejected": -0.038371045142412186, "step": 680 }, { "epoch": 0.51, "grad_norm": 0.33984375, "learning_rate": 2.838328249375328e-07, "logits/chosen": -2.0250961780548096, "logits/rejected": -1.5862263441085815, "logps/chosen": -230.3356475830078, "logps/rejected": -250.5562286376953, "loss": 0.6655, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.022846568375825882, "rewards/margins": 0.05219893902540207, "rewards/margins_max": 0.05219893902540207, "rewards/margins_min": 0.05219893902540207, "rewards/margins_std": NaN, "rewards/rejected": -0.029352372512221336, "step": 690 }, { "epoch": 0.52, "grad_norm": 0.3046875, "learning_rate": 2.774175148531329e-07, "logits/chosen": -2.0457088947296143, "logits/rejected": -1.532765507698059, "logps/chosen": -224.957763671875, "logps/rejected": -234.0838165283203, "loss": 0.6647, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.02778591774404049, "rewards/margins": 0.0492682009935379, "rewards/margins_max": 0.0492682009935379, "rewards/margins_min": 0.0492682009935379, "rewards/margins_std": NaN, "rewards/rejected": -0.021482281386852264, "step": 700 }, { "epoch": 0.53, "grad_norm": 0.2734375, "learning_rate": 2.7098387524922667e-07, "logits/chosen": -1.8189055919647217, "logits/rejected": -1.4590321779251099, "logps/chosen": -187.11215209960938, "logps/rejected": -179.0242156982422, "loss": 0.6658, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.01335279643535614, "rewards/margins": 0.04278566688299179, "rewards/margins_max": 0.04278566688299179, "rewards/margins_min": 0.04278566688299179, "rewards/margins_std": NaN, "rewards/rejected": -0.0294328723102808, "step": 710 }, { "epoch": 0.53, "grad_norm": 0.314453125, "learning_rate": 2.6453620722761895e-07, "logits/chosen": -2.1215291023254395, "logits/rejected": -1.4057071208953857, "logps/chosen": -323.5494689941406, "logps/rejected": -305.4034118652344, "loss": 0.6632, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.030355731025338173, "rewards/margins": 0.0672043040394783, "rewards/margins_max": 0.0672043040394783, "rewards/margins_min": 0.0672043040394783, "rewards/margins_std": NaN, "rewards/rejected": -0.036848582327365875, "step": 720 }, { "epoch": 0.54, "grad_norm": 0.32421875, "learning_rate": 2.580788212685777e-07, "logits/chosen": -2.018144369125366, "logits/rejected": -1.5041452646255493, "logps/chosen": -206.16226196289062, "logps/rejected": -224.09854125976562, "loss": 0.6658, "rewards/accuracies": 1.0, "rewards/chosen": 0.019517984241247177, "rewards/margins": 0.047711052000522614, "rewards/margins_max": 0.047711052000522614, "rewards/margins_min": 0.047711052000522614, "rewards/margins_std": NaN, "rewards/rejected": -0.028193065896630287, "step": 730 }, { "epoch": 0.55, "grad_norm": 0.30078125, "learning_rate": 2.516160343491348e-07, "logits/chosen": -2.0774478912353516, "logits/rejected": -1.3242664337158203, "logps/chosen": -261.04791259765625, "logps/rejected": -214.69058227539062, "loss": 0.6654, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.03108522854745388, "rewards/margins": 0.07123453170061111, "rewards/margins_max": 0.07123453170061111, "rewards/margins_min": 0.07123453170061111, "rewards/margins_std": NaN, "rewards/rejected": -0.040149301290512085, "step": 740 }, { "epoch": 0.56, "grad_norm": 0.337890625, "learning_rate": 2.4515216705704393e-07, "logits/chosen": -2.14136004447937, "logits/rejected": -1.2130528688430786, "logps/chosen": -302.6998291015625, "logps/rejected": -223.78067016601562, "loss": 0.6606, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.032542888075113297, "rewards/margins": 0.0818304568529129, "rewards/margins_max": 0.0818304568529129, "rewards/margins_min": 0.0818304568529129, "rewards/margins_std": NaN, "rewards/rejected": -0.04928756505250931, "step": 750 }, { "epoch": 0.56, "grad_norm": 0.328125, "learning_rate": 2.386915407023234e-07, "logits/chosen": -2.1050310134887695, "logits/rejected": -1.4062013626098633, "logps/chosen": -249.6138458251953, "logps/rejected": -318.61773681640625, "loss": 0.6605, "rewards/accuracies": 1.0, "rewards/chosen": 0.03660450130701065, "rewards/margins": 0.07939177751541138, "rewards/margins_max": 0.07939177751541138, "rewards/margins_min": 0.07939177751541138, "rewards/margins_std": NaN, "rewards/rejected": -0.042787276208400726, "step": 760 }, { "epoch": 0.57, "grad_norm": 0.296875, "learning_rate": 2.3223847442831753e-07, "logits/chosen": -2.0049426555633545, "logits/rejected": -1.5466521978378296, "logps/chosen": -226.28701782226562, "logps/rejected": -283.0534973144531, "loss": 0.6647, "rewards/accuracies": 1.0, "rewards/chosen": 0.01987474039196968, "rewards/margins": 0.07332300394773483, "rewards/margins_max": 0.07332300394773483, "rewards/margins_min": 0.07332300394773483, "rewards/margins_std": NaN, "rewards/rejected": -0.05344826728105545, "step": 770 }, { "epoch": 0.58, "grad_norm": 0.328125, "learning_rate": 2.2579728232420523e-07, "logits/chosen": -2.082897663116455, "logits/rejected": -1.373595118522644, "logps/chosen": -310.93695068359375, "logps/rejected": -240.2857208251953, "loss": 0.6632, "rewards/accuracies": 1.0, "rewards/chosen": 0.0339638814330101, "rewards/margins": 0.06445649266242981, "rewards/margins_max": 0.06445649266242981, "rewards/margins_min": 0.06445649266242981, "rewards/margins_std": NaN, "rewards/rejected": -0.030492613092064857, "step": 780 }, { "epoch": 0.58, "grad_norm": 0.380859375, "learning_rate": 2.1937227054088868e-07, "logits/chosen": -2.041259288787842, "logits/rejected": -1.4105521440505981, "logps/chosen": -291.784423828125, "logps/rejected": -246.40859985351562, "loss": 0.6643, "rewards/accuracies": 1.0, "rewards/chosen": 0.024001669138669968, "rewards/margins": 0.05560433119535446, "rewards/margins_max": 0.05560433119535446, "rewards/margins_min": 0.05560433119535446, "rewards/margins_std": NaN, "rewards/rejected": -0.031602658331394196, "step": 790 }, { "epoch": 0.59, "grad_norm": 0.318359375, "learning_rate": 2.1296773441218785e-07, "logits/chosen": -2.0651352405548096, "logits/rejected": -1.4306925535202026, "logps/chosen": -239.18588256835938, "logps/rejected": -210.4315948486328, "loss": 0.664, "rewards/accuracies": 1.0, "rewards/chosen": 0.026126444339752197, "rewards/margins": 0.06849897652864456, "rewards/margins_max": 0.06849897652864456, "rewards/margins_min": 0.06849897652864456, "rewards/margins_std": NaN, "rewards/rejected": -0.04237252101302147, "step": 800 }, { "epoch": 0.6, "grad_norm": 0.298828125, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.9278161525726318, "logits/rejected": -1.6418416500091553, "logps/chosen": -215.7371063232422, "logps/rejected": -255.38583374023438, "loss": 0.6652, "rewards/accuracies": 1.0, "rewards/chosen": 0.019038062542676926, "rewards/margins": 0.051104627549648285, "rewards/margins_max": 0.051104627549648285, "rewards/margins_min": 0.051104627549648285, "rewards/margins_std": NaN, "rewards/rejected": -0.03206656873226166, "step": 810 }, { "epoch": 0.61, "grad_norm": 0.34765625, "learning_rate": 2.0023719914821489e-07, "logits/chosen": -2.149594783782959, "logits/rejected": -1.4589303731918335, "logps/chosen": -219.695556640625, "logps/rejected": -241.76828002929688, "loss": 0.6628, "rewards/accuracies": 1.0, "rewards/chosen": 0.03597152605652809, "rewards/margins": 0.06724558025598526, "rewards/margins_max": 0.06724558025598526, "rewards/margins_min": 0.06724558025598526, "rewards/margins_std": NaN, "rewards/rejected": -0.03127405419945717, "step": 820 }, { "epoch": 0.61, "grad_norm": 0.310546875, "learning_rate": 1.9391971079868313e-07, "logits/chosen": -2.0435116291046143, "logits/rejected": -1.4584076404571533, "logps/chosen": -240.8225555419922, "logps/rejected": -303.6410827636719, "loss": 0.6637, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.029604297131299973, "rewards/margins": 0.06337974220514297, "rewards/margins_max": 0.06337974220514297, "rewards/margins_min": 0.06337974220514297, "rewards/margins_std": NaN, "rewards/rejected": -0.033775445073843, "step": 830 }, { "epoch": 0.62, "grad_norm": 0.314453125, "learning_rate": 1.8763971398550467e-07, "logits/chosen": -2.0277044773101807, "logits/rejected": -1.4923416376113892, "logps/chosen": -250.7859344482422, "logps/rejected": -254.98202514648438, "loss": 0.6673, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.028646240010857582, "rewards/margins": 0.054024677723646164, "rewards/margins_max": 0.054024677723646164, "rewards/margins_min": 0.054024677723646164, "rewards/margins_std": NaN, "rewards/rejected": -0.025378435850143433, "step": 840 }, { "epoch": 0.63, "grad_norm": 0.333984375, "learning_rate": 1.8140140709517465e-07, "logits/chosen": -2.047708034515381, "logits/rejected": -1.4980854988098145, "logps/chosen": -210.7559051513672, "logps/rejected": -207.94351196289062, "loss": 0.6643, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.01899396814405918, "rewards/margins": 0.048039522022008896, "rewards/margins_max": 0.048039522022008896, "rewards/margins_min": 0.048039522022008896, "rewards/margins_std": NaN, "rewards/rejected": -0.029045552015304565, "step": 850 }, { "epoch": 0.64, "grad_norm": 0.353515625, "learning_rate": 1.7520896064308963e-07, "logits/chosen": -1.8993380069732666, "logits/rejected": -1.4513304233551025, "logps/chosen": -223.9241943359375, "logps/rejected": -297.637939453125, "loss": 0.6628, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.018290331587195396, "rewards/margins": 0.0594097264111042, "rewards/margins_max": 0.0594097264111042, "rewards/margins_min": 0.0594097264111042, "rewards/margins_std": NaN, "rewards/rejected": -0.04111938923597336, "step": 860 }, { "epoch": 0.64, "grad_norm": 0.34765625, "learning_rate": 1.6906651448541976e-07, "logits/chosen": -2.0621097087860107, "logits/rejected": -1.237048625946045, "logps/chosen": -278.96832275390625, "logps/rejected": -267.96844482421875, "loss": 0.6608, "rewards/accuracies": 1.0, "rewards/chosen": 0.03597429022192955, "rewards/margins": 0.08543314039707184, "rewards/margins_max": 0.08543314039707184, "rewards/margins_min": 0.08543314039707184, "rewards/margins_std": NaN, "rewards/rejected": -0.04945885390043259, "step": 870 }, { "epoch": 0.65, "grad_norm": 0.298828125, "learning_rate": 1.6297817505147739e-07, "logits/chosen": -1.8318344354629517, "logits/rejected": -1.3975905179977417, "logps/chosen": -238.3777618408203, "logps/rejected": -247.81350708007812, "loss": 0.6627, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.02713797055184841, "rewards/margins": 0.06845160573720932, "rewards/margins_max": 0.06845160573720932, "rewards/margins_min": 0.06845160573720932, "rewards/margins_std": NaN, "rewards/rejected": -0.04131363332271576, "step": 880 }, { "epoch": 0.66, "grad_norm": 0.328125, "learning_rate": 1.5694801259843254e-07, "logits/chosen": -2.059532880783081, "logits/rejected": -1.5730485916137695, "logps/chosen": -219.37161254882812, "logps/rejected": -227.13394165039062, "loss": 0.6632, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.010322242975234985, "rewards/margins": 0.04914015904068947, "rewards/margins_max": 0.04914015904068947, "rewards/margins_min": 0.04914015904068947, "rewards/margins_std": NaN, "rewards/rejected": -0.03881791979074478, "step": 890 }, { "epoch": 0.67, "grad_norm": 0.357421875, "learning_rate": 1.5098005849021078e-07, "logits/chosen": -2.163090467453003, "logits/rejected": -1.4286366701126099, "logps/chosen": -239.53402709960938, "logps/rejected": -268.863037109375, "loss": 0.6635, "rewards/accuracies": 1.0, "rewards/chosen": 0.042647577822208405, "rewards/margins": 0.0607173927128315, "rewards/margins_max": 0.0607173927128315, "rewards/margins_min": 0.0607173927128315, "rewards/margins_std": NaN, "rewards/rejected": -0.018069813027977943, "step": 900 }, { "epoch": 0.67, "grad_norm": 0.4140625, "learning_rate": 1.4507830250239272e-07, "logits/chosen": -2.114851474761963, "logits/rejected": -1.3711202144622803, "logps/chosen": -214.02481079101562, "logps/rejected": -197.78500366210938, "loss": 0.665, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.03332282230257988, "rewards/margins": 0.057616226375103, "rewards/margins_max": 0.057616226375103, "rewards/margins_min": 0.057616226375103, "rewards/margins_std": NaN, "rewards/rejected": -0.02429339662194252, "step": 910 }, { "epoch": 0.68, "grad_norm": 0.32421875, "learning_rate": 1.3924669015491628e-07, "logits/chosen": -1.891593337059021, "logits/rejected": -1.4508168697357178, "logps/chosen": -190.78921508789062, "logps/rejected": -210.9984588623047, "loss": 0.6632, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.037054695188999176, "rewards/margins": 0.06240003556013107, "rewards/margins_max": 0.06240003556013107, "rewards/margins_min": 0.06240003556013107, "rewards/margins_std": NaN, "rewards/rejected": -0.025345345959067345, "step": 920 }, { "epoch": 0.69, "grad_norm": 0.322265625, "learning_rate": 1.3348912007436536e-07, "logits/chosen": -1.919196367263794, "logits/rejected": -1.4774754047393799, "logps/chosen": -243.7454376220703, "logps/rejected": -358.8896484375, "loss": 0.6585, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.021201713010668755, "rewards/margins": 0.0581810362637043, "rewards/margins_max": 0.0581810362637043, "rewards/margins_min": 0.0581810362637043, "rewards/margins_std": NaN, "rewards/rejected": -0.036979321390390396, "step": 930 }, { "epoch": 0.7, "grad_norm": 0.296875, "learning_rate": 1.2780944138760896e-07, "logits/chosen": -2.0747437477111816, "logits/rejected": -1.4782272577285767, "logps/chosen": -193.60910034179688, "logps/rejected": -173.97982788085938, "loss": 0.6677, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.020243193954229355, "rewards/margins": 0.04255754500627518, "rewards/margins_max": 0.04255754500627518, "rewards/margins_min": 0.04255754500627518, "rewards/margins_std": NaN, "rewards/rejected": -0.022314347326755524, "step": 940 }, { "epoch": 0.7, "grad_norm": 0.375, "learning_rate": 1.222114511485317e-07, "logits/chosen": -2.0575144290924072, "logits/rejected": -1.5969568490982056, "logps/chosen": -213.1886749267578, "logps/rejected": -241.73458862304688, "loss": 0.6621, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.037582967430353165, "rewards/margins": 0.061435867100954056, "rewards/margins_max": 0.061435867100954056, "rewards/margins_min": 0.061435867100954056, "rewards/margins_std": NaN, "rewards/rejected": -0.02385290339589119, "step": 950 }, { "epoch": 0.71, "grad_norm": 0.32421875, "learning_rate": 1.1669889179957723e-07, "logits/chosen": -1.9735885858535767, "logits/rejected": -1.662937879562378, "logps/chosen": -208.2716064453125, "logps/rejected": -174.0637664794922, "loss": 0.6654, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.009742580354213715, "rewards/margins": 0.03067430481314659, "rewards/margins_max": 0.03067430481314659, "rewards/margins_min": 0.03067430481314659, "rewards/margins_std": NaN, "rewards/rejected": -0.020931728184223175, "step": 960 }, { "epoch": 0.72, "grad_norm": 0.318359375, "learning_rate": 1.112754486698008e-07, "logits/chosen": -2.019521713256836, "logits/rejected": -1.5791001319885254, "logps/chosen": -187.46524047851562, "logps/rejected": -178.125732421875, "loss": 0.6615, "rewards/accuracies": 1.0, "rewards/chosen": 0.01672300137579441, "rewards/margins": 0.04681010916829109, "rewards/margins_max": 0.04681010916829109, "rewards/margins_min": 0.04681010916829109, "rewards/margins_std": NaN, "rewards/rejected": -0.03008710779249668, "step": 970 }, { "epoch": 0.73, "grad_norm": 0.373046875, "learning_rate": 1.059447475111049e-07, "logits/chosen": -1.9721561670303345, "logits/rejected": -1.4720313549041748, "logps/chosen": -247.0120849609375, "logps/rejected": -213.7181396484375, "loss": 0.6647, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.035904593765735626, "rewards/margins": 0.05502895638346672, "rewards/margins_max": 0.05502895638346672, "rewards/margins_min": 0.05502895638346672, "rewards/margins_std": NaN, "rewards/rejected": -0.019124364480376244, "step": 980 }, { "epoch": 0.73, "grad_norm": 0.326171875, "learning_rate": 1.007103520743035e-07, "logits/chosen": -2.0138068199157715, "logits/rejected": -1.4093111753463745, "logps/chosen": -254.36618041992188, "logps/rejected": -262.58319091796875, "loss": 0.6628, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.03567090258002281, "rewards/margins": 0.05842595174908638, "rewards/margins_max": 0.05842595174908638, "rewards/margins_min": 0.05842595174908638, "rewards/margins_std": NaN, "rewards/rejected": -0.022755051031708717, "step": 990 }, { "epoch": 0.74, "grad_norm": 0.30078125, "learning_rate": 9.557576172663575e-08, "logits/chosen": -1.9736032485961914, "logits/rejected": -1.531928300857544, "logps/chosen": -213.33682250976562, "logps/rejected": -193.63340759277344, "loss": 0.6648, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.012441306374967098, "rewards/margins": 0.04004935175180435, "rewards/margins_max": 0.04004935175180435, "rewards/margins_min": 0.04004935175180435, "rewards/margins_std": NaN, "rewards/rejected": -0.027608046308159828, "step": 1000 }, { "epoch": 0.75, "grad_norm": 0.34765625, "learning_rate": 9.054440911232347e-08, "logits/chosen": -1.9753551483154297, "logits/rejected": -1.4374115467071533, "logps/chosen": -257.1168518066406, "logps/rejected": -271.955078125, "loss": 0.6641, "rewards/accuracies": 1.0, "rewards/chosen": 0.018898367881774902, "rewards/margins": 0.054756682366132736, "rewards/margins_max": 0.054756682366132736, "rewards/margins_min": 0.054756682366132736, "rewards/margins_std": NaN, "rewards/rejected": -0.035858314484357834, "step": 1010 }, { "epoch": 0.76, "grad_norm": 0.30078125, "learning_rate": 8.561965785773412e-08, "logits/chosen": -2.133695125579834, "logits/rejected": -1.565525770187378, "logps/chosen": -212.60366821289062, "logps/rejected": -182.28314208984375, "loss": 0.6679, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.030873173847794533, "rewards/margins": 0.05696907639503479, "rewards/margins_max": 0.05696907639503479, "rewards/margins_min": 0.05696907639503479, "rewards/margins_std": NaN, "rewards/rejected": -0.026095900684595108, "step": 1020 }, { "epoch": 0.76, "grad_norm": 0.337890625, "learning_rate": 8.080480032268452e-08, "logits/chosen": -1.9859001636505127, "logits/rejected": -1.4256551265716553, "logps/chosen": -243.8011474609375, "logps/rejected": -223.94677734375, "loss": 0.6639, "rewards/accuracies": 1.0, "rewards/chosen": 0.02358115091919899, "rewards/margins": 0.051641564816236496, "rewards/margins_max": 0.051641564816236496, "rewards/margins_min": 0.051641564816236496, "rewards/margins_std": NaN, "rewards/rejected": -0.028060410171747208, "step": 1030 }, { "epoch": 0.77, "grad_norm": 0.322265625, "learning_rate": 7.61030553993895e-08, "logits/chosen": -1.9076287746429443, "logits/rejected": -1.4924620389938354, "logps/chosen": -216.83840942382812, "logps/rejected": -196.377685546875, "loss": 0.6633, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.013671179302036762, "rewards/margins": 0.049784958362579346, "rewards/margins_max": 0.049784958362579346, "rewards/margins_min": 0.049784958362579346, "rewards/margins_std": NaN, "rewards/rejected": -0.03611377626657486, "step": 1040 }, { "epoch": 0.78, "grad_norm": 0.314453125, "learning_rate": 7.151756636052527e-08, "logits/chosen": -2.130204916000366, "logits/rejected": -1.5611693859100342, "logps/chosen": -222.9783477783203, "logps/rejected": -241.9842987060547, "loss": 0.6591, "rewards/accuracies": 1.0, "rewards/chosen": 0.03606210649013519, "rewards/margins": 0.059976816177368164, "rewards/margins_max": 0.059976816177368164, "rewards/margins_min": 0.059976816177368164, "rewards/margins_std": NaN, "rewards/rejected": -0.02391471341252327, "step": 1050 }, { "epoch": 0.78, "grad_norm": 0.33984375, "learning_rate": 6.705139875784699e-08, "logits/chosen": -2.0021703243255615, "logits/rejected": -1.3691127300262451, "logps/chosen": -229.1208038330078, "logps/rejected": -196.5934295654297, "loss": 0.664, "rewards/accuracies": 1.0, "rewards/chosen": 0.016182031482458115, "rewards/margins": 0.04835697263479233, "rewards/margins_max": 0.04835697263479233, "rewards/margins_min": 0.04835697263479233, "rewards/margins_std": NaN, "rewards/rejected": -0.03217494487762451, "step": 1060 }, { "epoch": 0.79, "grad_norm": 0.357421875, "learning_rate": 6.270753837276599e-08, "logits/chosen": -1.9585247039794922, "logits/rejected": -1.6326916217803955, "logps/chosen": -208.2965850830078, "logps/rejected": -220.41714477539062, "loss": 0.6634, "rewards/accuracies": 1.0, "rewards/chosen": 0.02728954516351223, "rewards/margins": 0.05682883784174919, "rewards/margins_max": 0.05682883784174919, "rewards/margins_min": 0.05682883784174919, "rewards/margins_std": NaN, "rewards/rejected": -0.02953929826617241, "step": 1070 }, { "epoch": 0.8, "grad_norm": 0.322265625, "learning_rate": 5.848888922025552e-08, "logits/chosen": -2.0108110904693604, "logits/rejected": -1.5322282314300537, "logps/chosen": -193.75543212890625, "logps/rejected": -214.4615020751953, "loss": 0.6642, "rewards/accuracies": 1.0, "rewards/chosen": 0.02274712175130844, "rewards/margins": 0.062083650380373, "rewards/margins_max": 0.062083650380373, "rewards/margins_min": 0.062083650380373, "rewards/margins_std": NaN, "rewards/rejected": -0.03933652862906456, "step": 1080 }, { "epoch": 0.81, "grad_norm": 0.3125, "learning_rate": 5.439827160742006e-08, "logits/chosen": -1.9512097835540771, "logits/rejected": -1.6602758169174194, "logps/chosen": -189.49319458007812, "logps/rejected": -244.29592895507812, "loss": 0.6643, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.02437346801161766, "rewards/margins": 0.0436645932495594, "rewards/margins_max": 0.0436645932495594, "rewards/margins_min": 0.0436645932495594, "rewards/margins_std": NaN, "rewards/rejected": -0.019291121512651443, "step": 1090 }, { "epoch": 0.81, "grad_norm": 0.35546875, "learning_rate": 5.0438420248026745e-08, "logits/chosen": -1.8881117105484009, "logits/rejected": -1.3894041776657104, "logps/chosen": -198.81190490722656, "logps/rejected": -250.2027587890625, "loss": 0.6616, "rewards/accuracies": 1.0, "rewards/chosen": 0.03594060614705086, "rewards/margins": 0.07942356169223785, "rewards/margins_max": 0.07942356169223785, "rewards/margins_min": 0.07942356169223785, "rewards/margins_std": NaN, "rewards/rejected": -0.0434829518198967, "step": 1100 }, { "epoch": 0.82, "grad_norm": 0.34375, "learning_rate": 4.6611982434258124e-08, "logits/chosen": -1.934699296951294, "logits/rejected": -1.1928954124450684, "logps/chosen": -261.38677978515625, "logps/rejected": -241.5679473876953, "loss": 0.6629, "rewards/accuracies": 1.0, "rewards/chosen": 0.024552561342716217, "rewards/margins": 0.060813069343566895, "rewards/margins_max": 0.060813069343566895, "rewards/margins_min": 0.060813069343566895, "rewards/margins_std": NaN, "rewards/rejected": -0.03626050800085068, "step": 1110 }, { "epoch": 0.83, "grad_norm": 0.29296875, "learning_rate": 4.292151626690924e-08, "logits/chosen": -2.027463674545288, "logits/rejected": -1.3185806274414062, "logps/chosen": -252.04483032226562, "logps/rejected": -265.124267578125, "loss": 0.667, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.03300962597131729, "rewards/margins": 0.0655718669295311, "rewards/margins_max": 0.0655718669295311, "rewards/margins_min": 0.0655718669295311, "rewards/margins_std": NaN, "rewards/rejected": -0.032562240958213806, "step": 1120 }, { "epoch": 0.84, "grad_norm": 0.330078125, "learning_rate": 3.9369488945212606e-08, "logits/chosen": -2.0518107414245605, "logits/rejected": -1.5059715509414673, "logps/chosen": -211.740478515625, "logps/rejected": -182.64515686035156, "loss": 0.6653, "rewards/accuracies": 1.0, "rewards/chosen": 0.016033833846449852, "rewards/margins": 0.05374104902148247, "rewards/margins_max": 0.05374104902148247, "rewards/margins_min": 0.05374104902148247, "rewards/margins_std": NaN, "rewards/rejected": -0.037707217037677765, "step": 1130 }, { "epoch": 0.84, "grad_norm": 0.34375, "learning_rate": 3.5958275117433404e-08, "logits/chosen": -2.0312271118164062, "logits/rejected": -1.4592535495758057, "logps/chosen": -251.87002563476562, "logps/rejected": -245.94912719726562, "loss": 0.6674, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.025403957813978195, "rewards/margins": 0.058066435158252716, "rewards/margins_max": 0.058066435158252716, "rewards/margins_min": 0.058066435158252716, "rewards/margins_std": NaN, "rewards/rejected": -0.03266247734427452, "step": 1140 }, { "epoch": 0.85, "grad_norm": 0.361328125, "learning_rate": 3.269015529333805e-08, "logits/chosen": -2.043374538421631, "logits/rejected": -1.4708629846572876, "logps/chosen": -234.3373565673828, "logps/rejected": -253.6455841064453, "loss": 0.6632, "rewards/accuracies": 1.0, "rewards/chosen": 0.04083864018321037, "rewards/margins": 0.06850458681583405, "rewards/margins_max": 0.06850458681583405, "rewards/margins_min": 0.06850458681583405, "rewards/margins_std": NaN, "rewards/rejected": -0.027665939182043076, "step": 1150 }, { "epoch": 0.86, "grad_norm": 0.4296875, "learning_rate": 2.9567314319598e-08, "logits/chosen": -1.8975934982299805, "logits/rejected": -1.23898184299469, "logps/chosen": -194.3213348388672, "logps/rejected": -143.7144317626953, "loss": 0.6637, "rewards/accuracies": 1.0, "rewards/chosen": 0.029597630724310875, "rewards/margins": 0.058405883610248566, "rewards/margins_max": 0.058405883610248566, "rewards/margins_min": 0.058405883610248566, "rewards/margins_std": NaN, "rewards/rejected": -0.02880825474858284, "step": 1160 }, { "epoch": 0.87, "grad_norm": 0.30078125, "learning_rate": 2.659183991914696e-08, "logits/chosen": -2.1204023361206055, "logits/rejected": -1.3706982135772705, "logps/chosen": -244.99868774414062, "logps/rejected": -214.16195678710938, "loss": 0.6647, "rewards/accuracies": 1.0, "rewards/chosen": 0.02851070463657379, "rewards/margins": 0.07035355269908905, "rewards/margins_max": 0.07035355269908905, "rewards/margins_min": 0.07035355269908905, "rewards/margins_std": NaN, "rewards/rejected": -0.04184284433722496, "step": 1170 }, { "epoch": 0.87, "grad_norm": 0.310546875, "learning_rate": 2.3765721295468666e-08, "logits/chosen": -1.8714052438735962, "logits/rejected": -1.6353321075439453, "logps/chosen": -162.21560668945312, "logps/rejected": -206.0563507080078, "loss": 0.6639, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.027320072054862976, "rewards/margins": 0.04566752910614014, "rewards/margins_max": 0.04566752910614014, "rewards/margins_min": 0.04566752910614014, "rewards/margins_std": NaN, "rewards/rejected": -0.01834745705127716, "step": 1180 }, { "epoch": 0.88, "grad_norm": 0.35546875, "learning_rate": 2.109084780274853e-08, "logits/chosen": -1.9949709177017212, "logits/rejected": -1.4209120273590088, "logps/chosen": -259.18963623046875, "logps/rejected": -235.44589233398438, "loss": 0.6632, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.023489583283662796, "rewards/margins": 0.07520689070224762, "rewards/margins_max": 0.07520689070224762, "rewards/margins_min": 0.07520689070224762, "rewards/margins_std": NaN, "rewards/rejected": -0.05171731114387512, "step": 1190 }, { "epoch": 0.89, "grad_norm": 0.357421875, "learning_rate": 1.8569007682777415e-08, "logits/chosen": -2.0659101009368896, "logits/rejected": -1.2472288608551025, "logps/chosen": -237.00924682617188, "logps/rejected": -287.98492431640625, "loss": 0.6659, "rewards/accuracies": 1.0, "rewards/chosen": 0.032205354422330856, "rewards/margins": 0.06721274554729462, "rewards/margins_max": 0.06721274554729462, "rewards/margins_min": 0.06721274554729462, "rewards/margins_std": NaN, "rewards/rejected": -0.03500739112496376, "step": 1200 }, { "epoch": 0.9, "grad_norm": 0.32421875, "learning_rate": 1.620188686945259e-08, "logits/chosen": -2.077946186065674, "logits/rejected": -1.3815569877624512, "logps/chosen": -299.30889892578125, "logps/rejected": -249.3920135498047, "loss": 0.6653, "rewards/accuracies": 1.0, "rewards/chosen": 0.03304355591535568, "rewards/margins": 0.05800234526395798, "rewards/margins_max": 0.05800234526395798, "rewards/margins_min": 0.05800234526395798, "rewards/margins_std": NaN, "rewards/rejected": -0.024958789348602295, "step": 1210 }, { "epoch": 0.9, "grad_norm": 0.3125, "learning_rate": 1.3991067861675004e-08, "logits/chosen": -2.1188392639160156, "logits/rejected": -1.4769757986068726, "logps/chosen": -290.83831787109375, "logps/rejected": -317.05499267578125, "loss": 0.6598, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.02561381086707115, "rewards/margins": 0.05808825045824051, "rewards/margins_max": 0.05808825045824051, "rewards/margins_min": 0.05808825045824051, "rewards/margins_std": NaN, "rewards/rejected": -0.03247443586587906, "step": 1220 }, { "epoch": 0.91, "grad_norm": 0.302734375, "learning_rate": 1.1938028665396171e-08, "logits/chosen": -1.9317171573638916, "logits/rejected": -1.4349186420440674, "logps/chosen": -260.46673583984375, "logps/rejected": -225.98544311523438, "loss": 0.6616, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.025805342942476273, "rewards/margins": 0.05584897845983505, "rewards/margins_max": 0.05584897845983505, "rewards/margins_min": 0.05584897845983505, "rewards/margins_std": NaN, "rewards/rejected": -0.03004363179206848, "step": 1230 }, { "epoch": 0.92, "grad_norm": 0.291015625, "learning_rate": 1.0044141805522016e-08, "logits/chosen": -1.959181785583496, "logits/rejected": -1.532305359840393, "logps/chosen": -225.9596710205078, "logps/rejected": -205.9132537841797, "loss": 0.6615, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.021490924060344696, "rewards/margins": 0.04717742279171944, "rewards/margins_max": 0.04717742279171944, "rewards/margins_min": 0.04717742279171944, "rewards/margins_std": NaN, "rewards/rejected": -0.02568650245666504, "step": 1240 }, { "epoch": 0.93, "grad_norm": 0.30078125, "learning_rate": 8.310673408334496e-09, "logits/chosen": -2.0635626316070557, "logits/rejected": -1.3920619487762451, "logps/chosen": -252.69125366210938, "logps/rejected": -257.30242919921875, "loss": 0.6648, "rewards/accuracies": 1.0, "rewards/chosen": 0.036212801933288574, "rewards/margins": 0.061691027134656906, "rewards/margins_max": 0.061691027134656906, "rewards/margins_min": 0.061691027134656906, "rewards/margins_std": NaN, "rewards/rejected": -0.02547823265194893, "step": 1250 }, { "epoch": 0.93, "grad_norm": 0.373046875, "learning_rate": 6.738782355044048e-09, "logits/chosen": -2.0646793842315674, "logits/rejected": -1.4474830627441406, "logps/chosen": -206.7698516845703, "logps/rejected": -193.35592651367188, "loss": 0.6606, "rewards/accuracies": 1.0, "rewards/chosen": 0.023445401340723038, "rewards/margins": 0.059154510498046875, "rewards/margins_max": 0.059154510498046875, "rewards/margins_min": 0.059154510498046875, "rewards/margins_std": NaN, "rewards/rejected": -0.03570910915732384, "step": 1260 }, { "epoch": 0.94, "grad_norm": 0.349609375, "learning_rate": 5.329519507038971e-09, "logits/chosen": -1.9023844003677368, "logits/rejected": -1.439485788345337, "logps/chosen": -172.00807189941406, "logps/rejected": -252.7501983642578, "loss": 0.6673, "rewards/accuracies": 1.0, "rewards/chosen": 0.027050817385315895, "rewards/margins": 0.04781107231974602, "rewards/margins_max": 0.04781107231974602, "rewards/margins_min": 0.04781107231974602, "rewards/margins_std": NaN, "rewards/rejected": -0.02076025679707527, "step": 1270 }, { "epoch": 0.95, "grad_norm": 0.322265625, "learning_rate": 4.0838270033497215e-09, "logits/chosen": -1.969686508178711, "logits/rejected": -1.2621902227401733, "logps/chosen": -262.63665771484375, "logps/rejected": -258.04046630859375, "loss": 0.6626, "rewards/accuracies": 1.0, "rewards/chosen": 0.03954679146409035, "rewards/margins": 0.06421086192131042, "rewards/margins_max": 0.06421086192131042, "rewards/margins_min": 0.06421086192131042, "rewards/margins_std": NaN, "rewards/rejected": -0.024664074182510376, "step": 1280 }, { "epoch": 0.96, "grad_norm": 0.28515625, "learning_rate": 3.002537630797747e-09, "logits/chosen": -2.1349868774414062, "logits/rejected": -1.539178729057312, "logps/chosen": -239.86355590820312, "logps/rejected": -216.4762420654297, "loss": 0.6654, "rewards/accuracies": 1.0, "rewards/chosen": 0.02870868518948555, "rewards/margins": 0.05009246617555618, "rewards/margins_max": 0.05009246617555618, "rewards/margins_min": 0.05009246617555618, "rewards/margins_std": NaN, "rewards/rejected": -0.021383779123425484, "step": 1290 }, { "epoch": 0.96, "grad_norm": 0.2890625, "learning_rate": 2.086374267249724e-09, "logits/chosen": -1.9572795629501343, "logits/rejected": -1.5572397708892822, "logps/chosen": -175.27236938476562, "logps/rejected": -249.5878448486328, "loss": 0.6644, "rewards/accuracies": 1.0, "rewards/chosen": 0.03202231973409653, "rewards/margins": 0.0639900267124176, "rewards/margins_max": 0.0639900267124176, "rewards/margins_min": 0.0639900267124176, "rewards/margins_std": NaN, "rewards/rejected": -0.031967706978321075, "step": 1300 }, { "epoch": 0.97, "grad_norm": 0.31640625, "learning_rate": 1.335949398349634e-09, "logits/chosen": -1.8132435083389282, "logits/rejected": -1.541999101638794, "logps/chosen": -219.1054229736328, "logps/rejected": -244.1142120361328, "loss": 0.6644, "rewards/accuracies": 1.0, "rewards/chosen": 0.01563115045428276, "rewards/margins": 0.04497561603784561, "rewards/margins_max": 0.04497561603784561, "rewards/margins_min": 0.04497561603784561, "rewards/margins_std": NaN, "rewards/rejected": -0.02934446558356285, "step": 1310 }, { "epoch": 0.98, "grad_norm": 0.279296875, "learning_rate": 7.51764708051994e-10, "logits/chosen": -2.0751729011535645, "logits/rejected": -1.464297890663147, "logps/chosen": -209.7403564453125, "logps/rejected": -202.22215270996094, "loss": 0.6649, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.03106120601296425, "rewards/margins": 0.054163604974746704, "rewards/margins_max": 0.054163604974746704, "rewards/margins_min": 0.054163604974746704, "rewards/margins_std": NaN, "rewards/rejected": -0.023102398961782455, "step": 1320 }, { "epoch": 0.98, "grad_norm": 0.33203125, "learning_rate": 3.342107432292018e-10, "logits/chosen": -2.0309865474700928, "logits/rejected": -1.6503880023956299, "logps/chosen": -216.9345245361328, "logps/rejected": -258.69244384765625, "loss": 0.6634, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.028563106432557106, "rewards/margins": 0.058413028717041016, "rewards/margins_max": 0.058413028717041016, "rewards/margins_min": 0.058413028717041016, "rewards/margins_std": NaN, "rewards/rejected": -0.029849926009774208, "step": 1330 }, { "epoch": 0.99, "grad_norm": 0.359375, "learning_rate": 8.356665257813755e-11, "logits/chosen": -1.9864270687103271, "logits/rejected": -1.565781831741333, "logps/chosen": -220.52267456054688, "logps/rejected": -224.6304473876953, "loss": 0.6644, "rewards/accuracies": 1.0, "rewards/chosen": 0.028376713395118713, "rewards/margins": 0.06000853702425957, "rewards/margins_max": 0.06000853702425957, "rewards/margins_min": 0.06000853702425957, "rewards/margins_std": NaN, "rewards/rejected": -0.03163183480501175, "step": 1340 }, { "epoch": 1.0, "grad_norm": 0.337890625, "learning_rate": 0.0, "logits/chosen": -2.139542818069458, "logits/rejected": -1.3155004978179932, "logps/chosen": -288.415283203125, "logps/rejected": -208.6166229248047, "loss": 0.6643, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 0.03153330832719803, "rewards/margins": 0.05774186924099922, "rewards/margins_max": 0.05774186924099922, "rewards/margins_min": 0.05774186924099922, "rewards/margins_std": NaN, "rewards/rejected": -0.026208559051156044, "step": 1350 }, { "epoch": 1.0, "eval_logits/chosen": -1.1541073322296143, "eval_logits/rejected": -1.0117074251174927, "eval_logps/chosen": -321.7533264160156, "eval_logps/rejected": -313.631103515625, "eval_loss": 0.6924247145652771, "eval_rewards/accuracies": 0.5640000104904175, "eval_rewards/chosen": 0.023777291178703308, "eval_rewards/margins": 0.0020897758658975363, "eval_rewards/margins_max": 0.03932913392782211, "eval_rewards/margins_min": -0.036177705973386765, "eval_rewards/margins_std": 0.033525750041007996, "eval_rewards/rejected": 0.021687516942620277, "eval_runtime": 809.7327, "eval_samples_per_second": 9.88, "eval_steps_per_second": 0.309, "step": 1350 }, { "epoch": 1.0, "step": 1350, "total_flos": 0.0, "train_loss": 0.6709109875007911, "train_runtime": 7657.6869, "train_samples_per_second": 2.821, "train_steps_per_second": 0.176 } ], "logging_steps": 10, "max_steps": 1350, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 100, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }