diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,4150 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.9992254066615027, + "eval_steps": 100, + "global_step": 2904, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 1.7182130584192438e-09, + "logits/chosen": -2.067911148071289, + "logits/rejected": -1.9660799503326416, + "logps/chosen": -266.23834228515625, + "logps/rejected": -247.68406677246094, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.01, + "learning_rate": 1.718213058419244e-08, + "logits/chosen": -2.029080390930176, + "logits/rejected": -1.9344679117202759, + "logps/chosen": -291.5944519042969, + "logps/rejected": -213.14134216308594, + "loss": 0.6938, + "rewards/accuracies": 0.4652777910232544, + "rewards/chosen": 0.002623258624225855, + "rewards/margins": 0.0006023693131282926, + "rewards/rejected": 0.002020889427512884, + "step": 10 + }, + { + "epoch": 0.02, + "learning_rate": 3.436426116838488e-08, + "logits/chosen": -2.0757479667663574, + "logits/rejected": -1.9724565744400024, + "logps/chosen": -287.428955078125, + "logps/rejected": -239.43740844726562, + "loss": 0.6925, + "rewards/accuracies": 0.53125, + "rewards/chosen": 0.0022773859091103077, + "rewards/margins": 0.00433021504431963, + "rewards/rejected": -0.0020528293680399656, + "step": 20 + }, + { + "epoch": 0.03, + "learning_rate": 5.154639175257731e-08, + "logits/chosen": -2.0716583728790283, + "logits/rejected": -2.011603832244873, + "logps/chosen": -255.67135620117188, + "logps/rejected": -218.0836181640625, + "loss": 0.6949, + "rewards/accuracies": 0.5218750238418579, + "rewards/chosen": -0.0011378427734598517, + "rewards/margins": -0.0006913787801750004, + "rewards/rejected": -0.0004464639350771904, + "step": 30 + }, + { + "epoch": 0.04, + "learning_rate": 6.872852233676976e-08, + "logits/chosen": -2.079998731613159, + "logits/rejected": -2.0144975185394287, + "logps/chosen": -276.287353515625, + "logps/rejected": -230.61965942382812, + "loss": 0.6944, + "rewards/accuracies": 0.484375, + "rewards/chosen": -0.0009711446473374963, + "rewards/margins": -3.942637704312801e-06, + "rewards/rejected": -0.0009672018932178617, + "step": 40 + }, + { + "epoch": 0.05, + "learning_rate": 8.59106529209622e-08, + "logits/chosen": -1.9645802974700928, + "logits/rejected": -1.873591661453247, + "logps/chosen": -269.802490234375, + "logps/rejected": -218.36865234375, + "loss": 0.6929, + "rewards/accuracies": 0.534375011920929, + "rewards/chosen": 0.003975920379161835, + "rewards/margins": 0.0016158551443368196, + "rewards/rejected": 0.002360065234825015, + "step": 50 + }, + { + "epoch": 0.06, + "learning_rate": 1.0309278350515462e-07, + "logits/chosen": -1.9920260906219482, + "logits/rejected": -2.014172077178955, + "logps/chosen": -263.1590270996094, + "logps/rejected": -230.80264282226562, + "loss": 0.6923, + "rewards/accuracies": 0.534375011920929, + "rewards/chosen": 0.0037864961195737123, + "rewards/margins": 0.0038312277756631374, + "rewards/rejected": -4.4732307287631556e-05, + "step": 60 + }, + { + "epoch": 0.07, + "learning_rate": 1.202749140893471e-07, + "logits/chosen": -2.0915229320526123, + "logits/rejected": -1.9606736898422241, + "logps/chosen": -292.42498779296875, + "logps/rejected": -242.525146484375, + "loss": 0.6927, + "rewards/accuracies": 0.48124998807907104, + "rewards/chosen": -0.0015628642868250608, + "rewards/margins": -0.0018646640237420797, + "rewards/rejected": 0.0003017998533323407, + "step": 70 + }, + { + "epoch": 0.08, + "learning_rate": 1.3745704467353952e-07, + "logits/chosen": -2.064213275909424, + "logits/rejected": -1.9719228744506836, + "logps/chosen": -279.91070556640625, + "logps/rejected": -226.50143432617188, + "loss": 0.6928, + "rewards/accuracies": 0.49687498807907104, + "rewards/chosen": -0.002248315839096904, + "rewards/margins": 0.00012451801740098745, + "rewards/rejected": -0.002372834598645568, + "step": 80 + }, + { + "epoch": 0.09, + "learning_rate": 1.5463917525773197e-07, + "logits/chosen": -1.9888744354248047, + "logits/rejected": -1.9386621713638306, + "logps/chosen": -261.8848571777344, + "logps/rejected": -214.2974395751953, + "loss": 0.6927, + "rewards/accuracies": 0.503125011920929, + "rewards/chosen": -0.001680875546298921, + "rewards/margins": -2.3123900973587297e-05, + "rewards/rejected": -0.001657751388847828, + "step": 90 + }, + { + "epoch": 0.1, + "learning_rate": 1.718213058419244e-07, + "logits/chosen": -1.9539144039154053, + "logits/rejected": -1.8788636922836304, + "logps/chosen": -271.3200378417969, + "logps/rejected": -216.4374542236328, + "loss": 0.6922, + "rewards/accuracies": 0.534375011920929, + "rewards/chosen": 0.0035204023588448763, + "rewards/margins": 0.0032666183542460203, + "rewards/rejected": 0.0002537833934184164, + "step": 100 + }, + { + "epoch": 0.11, + "learning_rate": 1.8900343642611682e-07, + "logits/chosen": -2.0144975185394287, + "logits/rejected": -1.870060682296753, + "logps/chosen": -256.0116271972656, + "logps/rejected": -214.73751831054688, + "loss": 0.6904, + "rewards/accuracies": 0.5218750238418579, + "rewards/chosen": 0.0029162573628127575, + "rewards/margins": 0.004548544529825449, + "rewards/rejected": -0.0016322873998433352, + "step": 110 + }, + { + "epoch": 0.12, + "learning_rate": 2.0618556701030925e-07, + "logits/chosen": -2.0126750469207764, + "logits/rejected": -1.9877809286117554, + "logps/chosen": -294.35772705078125, + "logps/rejected": -230.9095916748047, + "loss": 0.6898, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": 0.006383317522704601, + "rewards/margins": 0.010343357920646667, + "rewards/rejected": -0.003960040397942066, + "step": 120 + }, + { + "epoch": 0.13, + "learning_rate": 2.2336769759450173e-07, + "logits/chosen": -2.039785385131836, + "logits/rejected": -1.9796234369277954, + "logps/chosen": -270.469970703125, + "logps/rejected": -223.00634765625, + "loss": 0.6921, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": 0.008164801634848118, + "rewards/margins": 0.00915917381644249, + "rewards/rejected": -0.0009943728800863028, + "step": 130 + }, + { + "epoch": 0.14, + "learning_rate": 2.405498281786942e-07, + "logits/chosen": -2.1231284141540527, + "logits/rejected": -2.010619640350342, + "logps/chosen": -290.5582580566406, + "logps/rejected": -214.4351043701172, + "loss": 0.6908, + "rewards/accuracies": 0.528124988079071, + "rewards/chosen": 0.004269259050488472, + "rewards/margins": 0.007282177917659283, + "rewards/rejected": -0.0030129191000014544, + "step": 140 + }, + { + "epoch": 0.15, + "learning_rate": 2.5773195876288655e-07, + "logits/chosen": -2.0174431800842285, + "logits/rejected": -2.000694751739502, + "logps/chosen": -267.80718994140625, + "logps/rejected": -228.5482177734375, + "loss": 0.6879, + "rewards/accuracies": 0.543749988079071, + "rewards/chosen": 0.007982608862221241, + "rewards/margins": 0.00814131461083889, + "rewards/rejected": -0.00015870490460656583, + "step": 150 + }, + { + "epoch": 0.17, + "learning_rate": 2.7491408934707903e-07, + "logits/chosen": -2.1166934967041016, + "logits/rejected": -2.0119781494140625, + "logps/chosen": -268.8128356933594, + "logps/rejected": -215.2762451171875, + "loss": 0.6871, + "rewards/accuracies": 0.6156250238418579, + "rewards/chosen": 0.006122085265815258, + "rewards/margins": 0.012544795870780945, + "rewards/rejected": -0.006422711070626974, + "step": 160 + }, + { + "epoch": 0.18, + "learning_rate": 2.9209621993127146e-07, + "logits/chosen": -2.031827211380005, + "logits/rejected": -2.0032095909118652, + "logps/chosen": -270.9142150878906, + "logps/rejected": -227.9224090576172, + "loss": 0.6851, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": 0.012479735538363457, + "rewards/margins": 0.015380832366645336, + "rewards/rejected": -0.0029010965954512358, + "step": 170 + }, + { + "epoch": 0.19, + "learning_rate": 3.0927835051546394e-07, + "logits/chosen": -2.0390381813049316, + "logits/rejected": -1.9806989431381226, + "logps/chosen": -251.3668975830078, + "logps/rejected": -210.38339233398438, + "loss": 0.6836, + "rewards/accuracies": 0.621874988079071, + "rewards/chosen": 0.01502176932990551, + "rewards/margins": 0.022143321111798286, + "rewards/rejected": -0.007121548987925053, + "step": 180 + }, + { + "epoch": 0.2, + "learning_rate": 3.2646048109965636e-07, + "logits/chosen": -2.027550220489502, + "logits/rejected": -1.9014009237289429, + "logps/chosen": -264.51934814453125, + "logps/rejected": -213.80886840820312, + "loss": 0.6818, + "rewards/accuracies": 0.6343749761581421, + "rewards/chosen": 0.015689093619585037, + "rewards/margins": 0.027370622381567955, + "rewards/rejected": -0.011681526899337769, + "step": 190 + }, + { + "epoch": 0.21, + "learning_rate": 3.436426116838488e-07, + "logits/chosen": -2.034299850463867, + "logits/rejected": -1.9709196090698242, + "logps/chosen": -273.13226318359375, + "logps/rejected": -211.93576049804688, + "loss": 0.6807, + "rewards/accuracies": 0.621874988079071, + "rewards/chosen": 0.016958530992269516, + "rewards/margins": 0.023664183914661407, + "rewards/rejected": -0.0067056515254080296, + "step": 200 + }, + { + "epoch": 0.22, + "learning_rate": 3.608247422680412e-07, + "logits/chosen": -2.0183119773864746, + "logits/rejected": -1.9083303213119507, + "logps/chosen": -275.7691345214844, + "logps/rejected": -208.54074096679688, + "loss": 0.6773, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": 0.01859709620475769, + "rewards/margins": 0.031639259308576584, + "rewards/rejected": -0.013042164035141468, + "step": 210 + }, + { + "epoch": 0.23, + "learning_rate": 3.7800687285223364e-07, + "logits/chosen": -2.016589879989624, + "logits/rejected": -1.9477627277374268, + "logps/chosen": -259.4232177734375, + "logps/rejected": -226.46463012695312, + "loss": 0.6769, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": 0.024477588012814522, + "rewards/margins": 0.03540473431348801, + "rewards/rejected": -0.010927150025963783, + "step": 220 + }, + { + "epoch": 0.24, + "learning_rate": 3.9518900343642607e-07, + "logits/chosen": -2.0396270751953125, + "logits/rejected": -1.9370222091674805, + "logps/chosen": -257.0804748535156, + "logps/rejected": -235.4674835205078, + "loss": 0.6789, + "rewards/accuracies": 0.6343749761581421, + "rewards/chosen": 0.023364732041954994, + "rewards/margins": 0.03109937347471714, + "rewards/rejected": -0.0077346437610685825, + "step": 230 + }, + { + "epoch": 0.25, + "learning_rate": 4.123711340206185e-07, + "logits/chosen": -1.9943723678588867, + "logits/rejected": -1.8853381872177124, + "logps/chosen": -273.07086181640625, + "logps/rejected": -206.6876983642578, + "loss": 0.6748, + "rewards/accuracies": 0.6312500238418579, + "rewards/chosen": 0.022974053397774696, + "rewards/margins": 0.03632950782775879, + "rewards/rejected": -0.013355454429984093, + "step": 240 + }, + { + "epoch": 0.26, + "learning_rate": 4.2955326460481097e-07, + "logits/chosen": -2.0440022945404053, + "logits/rejected": -1.9706653356552124, + "logps/chosen": -280.75225830078125, + "logps/rejected": -229.0762939453125, + "loss": 0.673, + "rewards/accuracies": 0.6031249761581421, + "rewards/chosen": 0.02196577563881874, + "rewards/margins": 0.03916791081428528, + "rewards/rejected": -0.017202135175466537, + "step": 250 + }, + { + "epoch": 0.27, + "learning_rate": 4.4673539518900345e-07, + "logits/chosen": -1.9852807521820068, + "logits/rejected": -1.913886308670044, + "logps/chosen": -251.1424102783203, + "logps/rejected": -209.325927734375, + "loss": 0.6707, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": 0.027156567201018333, + "rewards/margins": 0.04766073822975159, + "rewards/rejected": -0.020504174754023552, + "step": 260 + }, + { + "epoch": 0.28, + "learning_rate": 4.639175257731959e-07, + "logits/chosen": -2.0986721515655518, + "logits/rejected": -2.0114688873291016, + "logps/chosen": -259.94476318359375, + "logps/rejected": -216.35888671875, + "loss": 0.6658, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": 0.03609105944633484, + "rewards/margins": 0.06675679981708527, + "rewards/rejected": -0.030665744096040726, + "step": 270 + }, + { + "epoch": 0.29, + "learning_rate": 4.810996563573884e-07, + "logits/chosen": -2.023041248321533, + "logits/rejected": -1.9758949279785156, + "logps/chosen": -279.7530212402344, + "logps/rejected": -239.0085906982422, + "loss": 0.6637, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": 0.0377194806933403, + "rewards/margins": 0.07151970267295837, + "rewards/rejected": -0.03380022197961807, + "step": 280 + }, + { + "epoch": 0.3, + "learning_rate": 4.982817869415807e-07, + "logits/chosen": -1.9995616674423218, + "logits/rejected": -1.9275434017181396, + "logps/chosen": -255.81723022460938, + "logps/rejected": -212.13525390625, + "loss": 0.6633, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": 0.0429212860763073, + "rewards/margins": 0.07059122622013092, + "rewards/rejected": -0.027669942006468773, + "step": 290 + }, + { + "epoch": 0.31, + "learning_rate": 4.982778415614236e-07, + "logits/chosen": -1.9670026302337646, + "logits/rejected": -1.9164260625839233, + "logps/chosen": -247.3227996826172, + "logps/rejected": -229.3400115966797, + "loss": 0.6599, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": 0.041430626064538956, + "rewards/margins": 0.06908479332923889, + "rewards/rejected": -0.027654165402054787, + "step": 300 + }, + { + "epoch": 0.32, + "learning_rate": 4.963643321852277e-07, + "logits/chosen": -2.008664131164551, + "logits/rejected": -2.0269663333892822, + "logps/chosen": -278.21331787109375, + "logps/rejected": -230.90005493164062, + "loss": 0.6557, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": 0.04368427395820618, + "rewards/margins": 0.09482843428850174, + "rewards/rejected": -0.05114416405558586, + "step": 310 + }, + { + "epoch": 0.33, + "learning_rate": 4.944508228090318e-07, + "logits/chosen": -2.0187923908233643, + "logits/rejected": -1.9102197885513306, + "logps/chosen": -257.12445068359375, + "logps/rejected": -189.22702026367188, + "loss": 0.645, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": 0.05188002064824104, + "rewards/margins": 0.10916055738925934, + "rewards/rejected": -0.057280540466308594, + "step": 320 + }, + { + "epoch": 0.34, + "learning_rate": 4.925373134328357e-07, + "logits/chosen": -2.0044243335723877, + "logits/rejected": -1.9528205394744873, + "logps/chosen": -269.754150390625, + "logps/rejected": -210.67074584960938, + "loss": 0.6513, + "rewards/accuracies": 0.653124988079071, + "rewards/chosen": 0.05257422477006912, + "rewards/margins": 0.09687207639217377, + "rewards/rejected": -0.04429786279797554, + "step": 330 + }, + { + "epoch": 0.35, + "learning_rate": 4.906238040566398e-07, + "logits/chosen": -2.006479263305664, + "logits/rejected": -1.8888683319091797, + "logps/chosen": -239.5758819580078, + "logps/rejected": -227.4214630126953, + "loss": 0.6485, + "rewards/accuracies": 0.6656249761581421, + "rewards/chosen": 0.039284877479076385, + "rewards/margins": 0.08342822641134262, + "rewards/rejected": -0.04414334148168564, + "step": 340 + }, + { + "epoch": 0.36, + "learning_rate": 4.887102946804438e-07, + "logits/chosen": -2.125704288482666, + "logits/rejected": -2.004089117050171, + "logps/chosen": -290.8746337890625, + "logps/rejected": -230.57009887695312, + "loss": 0.6404, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": 0.06995740532875061, + "rewards/margins": 0.1391131579875946, + "rewards/rejected": -0.0691557452082634, + "step": 350 + }, + { + "epoch": 0.37, + "learning_rate": 4.867967853042479e-07, + "logits/chosen": -2.009556770324707, + "logits/rejected": -2.0069127082824707, + "logps/chosen": -254.616943359375, + "logps/rejected": -215.4530792236328, + "loss": 0.6299, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": 0.05212836712598801, + "rewards/margins": 0.15012125670909882, + "rewards/rejected": -0.09799288213253021, + "step": 360 + }, + { + "epoch": 0.38, + "learning_rate": 4.84883275928052e-07, + "logits/chosen": -2.1299102306365967, + "logits/rejected": -1.983077049255371, + "logps/chosen": -284.6875305175781, + "logps/rejected": -251.5108642578125, + "loss": 0.6313, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": 0.08003100007772446, + "rewards/margins": 0.1480170488357544, + "rewards/rejected": -0.06798602640628815, + "step": 370 + }, + { + "epoch": 0.39, + "learning_rate": 4.82969766551856e-07, + "logits/chosen": -2.0552008152008057, + "logits/rejected": -1.9739630222320557, + "logps/chosen": -276.0761413574219, + "logps/rejected": -239.1278839111328, + "loss": 0.6369, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.06775949150323868, + "rewards/margins": 0.1486402004957199, + "rewards/rejected": -0.08088071644306183, + "step": 380 + }, + { + "epoch": 0.4, + "learning_rate": 4.810562571756601e-07, + "logits/chosen": -2.0520501136779785, + "logits/rejected": -1.945320725440979, + "logps/chosen": -284.12017822265625, + "logps/rejected": -231.8722686767578, + "loss": 0.6302, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": 0.06102215498685837, + "rewards/margins": 0.17737627029418945, + "rewards/rejected": -0.11635412275791168, + "step": 390 + }, + { + "epoch": 0.41, + "learning_rate": 4.791427477994642e-07, + "logits/chosen": -2.008812189102173, + "logits/rejected": -1.9848194122314453, + "logps/chosen": -277.79376220703125, + "logps/rejected": -224.36083984375, + "loss": 0.6243, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": 0.0664195567369461, + "rewards/margins": 0.17244522273540497, + "rewards/rejected": -0.10602565854787827, + "step": 400 + }, + { + "epoch": 0.42, + "learning_rate": 4.772292384232682e-07, + "logits/chosen": -2.009459972381592, + "logits/rejected": -1.9265193939208984, + "logps/chosen": -263.84259033203125, + "logps/rejected": -243.3984832763672, + "loss": 0.621, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": 0.056257862597703934, + "rewards/margins": 0.17955340445041656, + "rewards/rejected": -0.12329553067684174, + "step": 410 + }, + { + "epoch": 0.43, + "learning_rate": 4.753157290470723e-07, + "logits/chosen": -2.0507168769836426, + "logits/rejected": -1.961004614830017, + "logps/chosen": -253.4503173828125, + "logps/rejected": -205.6368408203125, + "loss": 0.6309, + "rewards/accuracies": 0.6187499761581421, + "rewards/chosen": 0.02212332747876644, + "rewards/margins": 0.1393744796514511, + "rewards/rejected": -0.11725115776062012, + "step": 420 + }, + { + "epoch": 0.44, + "learning_rate": 4.7340221967087635e-07, + "logits/chosen": -2.042525291442871, + "logits/rejected": -1.9645607471466064, + "logps/chosen": -264.80645751953125, + "logps/rejected": -226.8705596923828, + "loss": 0.6296, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": 0.02932189777493477, + "rewards/margins": 0.15064987540245056, + "rewards/rejected": -0.12132799625396729, + "step": 430 + }, + { + "epoch": 0.45, + "learning_rate": 4.714887102946804e-07, + "logits/chosen": -2.0784430503845215, + "logits/rejected": -1.9688632488250732, + "logps/chosen": -276.26812744140625, + "logps/rejected": -225.707275390625, + "loss": 0.6125, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": 0.05800630897283554, + "rewards/margins": 0.21699848771095276, + "rewards/rejected": -0.15899215638637543, + "step": 440 + }, + { + "epoch": 0.46, + "learning_rate": 4.6957520091848447e-07, + "logits/chosen": -1.9616053104400635, + "logits/rejected": -1.991091012954712, + "logps/chosen": -256.5301208496094, + "logps/rejected": -231.73568725585938, + "loss": 0.6361, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": 0.03827190026640892, + "rewards/margins": 0.13751840591430664, + "rewards/rejected": -0.09924649447202682, + "step": 450 + }, + { + "epoch": 0.48, + "learning_rate": 4.6766169154228853e-07, + "logits/chosen": -2.022930145263672, + "logits/rejected": -1.9804435968399048, + "logps/chosen": -263.57073974609375, + "logps/rejected": -228.65420532226562, + "loss": 0.5982, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": 0.049893591552972794, + "rewards/margins": 0.2338247299194336, + "rewards/rejected": -0.1839311420917511, + "step": 460 + }, + { + "epoch": 0.49, + "learning_rate": 4.657481821660926e-07, + "logits/chosen": -2.0467746257781982, + "logits/rejected": -1.957139015197754, + "logps/chosen": -251.3452911376953, + "logps/rejected": -205.276611328125, + "loss": 0.6046, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": 0.07499710470438004, + "rewards/margins": 0.266188383102417, + "rewards/rejected": -0.19119124114513397, + "step": 470 + }, + { + "epoch": 0.5, + "learning_rate": 4.6383467278989666e-07, + "logits/chosen": -2.097900867462158, + "logits/rejected": -1.9721593856811523, + "logps/chosen": -262.32122802734375, + "logps/rejected": -229.8935546875, + "loss": 0.6135, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": 0.04003770649433136, + "rewards/margins": 0.23067660629749298, + "rewards/rejected": -0.19063889980316162, + "step": 480 + }, + { + "epoch": 0.51, + "learning_rate": 4.6192116341370067e-07, + "logits/chosen": -2.130352258682251, + "logits/rejected": -1.9925979375839233, + "logps/chosen": -264.1942443847656, + "logps/rejected": -227.0887908935547, + "loss": 0.5973, + "rewards/accuracies": 0.703125, + "rewards/chosen": 0.06981977075338364, + "rewards/margins": 0.277499794960022, + "rewards/rejected": -0.20768003165721893, + "step": 490 + }, + { + "epoch": 0.52, + "learning_rate": 4.6000765403750473e-07, + "logits/chosen": -2.038247585296631, + "logits/rejected": -1.940549612045288, + "logps/chosen": -263.71923828125, + "logps/rejected": -212.58712768554688, + "loss": 0.5982, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": 0.09329665452241898, + "rewards/margins": 0.28916245698928833, + "rewards/rejected": -0.19586579501628876, + "step": 500 + }, + { + "epoch": 0.53, + "learning_rate": 4.580941446613088e-07, + "logits/chosen": -1.9596498012542725, + "logits/rejected": -1.9526113271713257, + "logps/chosen": -270.47509765625, + "logps/rejected": -229.74697875976562, + "loss": 0.598, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.053180061280727386, + "rewards/margins": 0.26379942893981934, + "rewards/rejected": -0.21061936020851135, + "step": 510 + }, + { + "epoch": 0.54, + "learning_rate": 4.5618063528511285e-07, + "logits/chosen": -2.062441825866699, + "logits/rejected": -1.9531586170196533, + "logps/chosen": -262.12518310546875, + "logps/rejected": -221.9291229248047, + "loss": 0.604, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": 0.052028585225343704, + "rewards/margins": 0.2801084518432617, + "rewards/rejected": -0.2280798852443695, + "step": 520 + }, + { + "epoch": 0.55, + "learning_rate": 4.542671259089169e-07, + "logits/chosen": -1.9766355752944946, + "logits/rejected": -1.936336874961853, + "logps/chosen": -235.9636993408203, + "logps/rejected": -208.9865264892578, + "loss": 0.6075, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": 0.024395454674959183, + "rewards/margins": 0.23684442043304443, + "rewards/rejected": -0.21244895458221436, + "step": 530 + }, + { + "epoch": 0.56, + "learning_rate": 4.52353616532721e-07, + "logits/chosen": -2.052635908126831, + "logits/rejected": -1.9279887676239014, + "logps/chosen": -272.1145324707031, + "logps/rejected": -228.16616821289062, + "loss": 0.6071, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": 0.06684047728776932, + "rewards/margins": 0.3025699555873871, + "rewards/rejected": -0.23572945594787598, + "step": 540 + }, + { + "epoch": 0.57, + "learning_rate": 4.5044010715652504e-07, + "logits/chosen": -2.086524248123169, + "logits/rejected": -1.9210243225097656, + "logps/chosen": -265.66644287109375, + "logps/rejected": -231.428466796875, + "loss": 0.6064, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": 0.044201988726854324, + "rewards/margins": 0.2510104179382324, + "rewards/rejected": -0.2068084180355072, + "step": 550 + }, + { + "epoch": 0.58, + "learning_rate": 4.485265977803291e-07, + "logits/chosen": -2.052487850189209, + "logits/rejected": -1.963782548904419, + "logps/chosen": -274.49444580078125, + "logps/rejected": -228.4085693359375, + "loss": 0.5943, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": 0.07493807375431061, + "rewards/margins": 0.3155871033668518, + "rewards/rejected": -0.2406490296125412, + "step": 560 + }, + { + "epoch": 0.59, + "learning_rate": 4.4661308840413316e-07, + "logits/chosen": -2.0338144302368164, + "logits/rejected": -1.9587665796279907, + "logps/chosen": -292.2133483886719, + "logps/rejected": -242.80636596679688, + "loss": 0.598, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": 0.07242296636104584, + "rewards/margins": 0.331615149974823, + "rewards/rejected": -0.25919219851493835, + "step": 570 + }, + { + "epoch": 0.6, + "learning_rate": 4.446995790279372e-07, + "logits/chosen": -2.022578477859497, + "logits/rejected": -1.9327418804168701, + "logps/chosen": -275.12005615234375, + "logps/rejected": -230.9717559814453, + "loss": 0.5784, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": 0.08595068752765656, + "rewards/margins": 0.37252914905548096, + "rewards/rejected": -0.2865784466266632, + "step": 580 + }, + { + "epoch": 0.61, + "learning_rate": 4.4278606965174123e-07, + "logits/chosen": -2.0303807258605957, + "logits/rejected": -1.9269481897354126, + "logps/chosen": -267.8510437011719, + "logps/rejected": -225.8118896484375, + "loss": 0.5948, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 0.05344559997320175, + "rewards/margins": 0.3274499475955963, + "rewards/rejected": -0.2740043103694916, + "step": 590 + }, + { + "epoch": 0.62, + "learning_rate": 4.408725602755453e-07, + "logits/chosen": -2.0389864444732666, + "logits/rejected": -1.9150155782699585, + "logps/chosen": -240.3448028564453, + "logps/rejected": -215.1417694091797, + "loss": 0.6007, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": 0.01124047301709652, + "rewards/margins": 0.26555973291397095, + "rewards/rejected": -0.2543192505836487, + "step": 600 + }, + { + "epoch": 0.63, + "learning_rate": 4.3895905089934936e-07, + "logits/chosen": -1.9739296436309814, + "logits/rejected": -1.8714065551757812, + "logps/chosen": -251.74917602539062, + "logps/rejected": -219.4097900390625, + "loss": 0.5993, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.04064112901687622, + "rewards/margins": 0.270423948764801, + "rewards/rejected": -0.22978278994560242, + "step": 610 + }, + { + "epoch": 0.64, + "learning_rate": 4.370455415231534e-07, + "logits/chosen": -2.0759198665618896, + "logits/rejected": -1.94767165184021, + "logps/chosen": -309.49407958984375, + "logps/rejected": -251.93539428710938, + "loss": 0.5811, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": 0.08607195317745209, + "rewards/margins": 0.3619040548801422, + "rewards/rejected": -0.2758321166038513, + "step": 620 + }, + { + "epoch": 0.65, + "learning_rate": 4.351320321469575e-07, + "logits/chosen": -2.035043239593506, + "logits/rejected": -1.9782638549804688, + "logps/chosen": -262.66717529296875, + "logps/rejected": -229.33609008789062, + "loss": 0.591, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.05298718065023422, + "rewards/margins": 0.3540084660053253, + "rewards/rejected": -0.3010213077068329, + "step": 630 + }, + { + "epoch": 0.66, + "learning_rate": 4.3321852277076154e-07, + "logits/chosen": -2.0530083179473877, + "logits/rejected": -1.959703803062439, + "logps/chosen": -257.1555480957031, + "logps/rejected": -225.45669555664062, + "loss": 0.5719, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": 0.018595661967992783, + "rewards/margins": 0.3380756974220276, + "rewards/rejected": -0.3194800615310669, + "step": 640 + }, + { + "epoch": 0.67, + "learning_rate": 4.313050133945656e-07, + "logits/chosen": -2.026218891143799, + "logits/rejected": -1.9713115692138672, + "logps/chosen": -266.67706298828125, + "logps/rejected": -217.25247192382812, + "loss": 0.5864, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.06421274691820145, + "rewards/margins": 0.4028696119785309, + "rewards/rejected": -0.33865687251091003, + "step": 650 + }, + { + "epoch": 0.68, + "learning_rate": 4.2939150401836967e-07, + "logits/chosen": -2.0551767349243164, + "logits/rejected": -1.9555637836456299, + "logps/chosen": -277.0030212402344, + "logps/rejected": -237.25796508789062, + "loss": 0.5773, + "rewards/accuracies": 0.671875, + "rewards/chosen": 0.0361778661608696, + "rewards/margins": 0.36955034732818604, + "rewards/rejected": -0.3333725035190582, + "step": 660 + }, + { + "epoch": 0.69, + "learning_rate": 4.2747799464217373e-07, + "logits/chosen": -2.0053582191467285, + "logits/rejected": -1.964155912399292, + "logps/chosen": -274.88311767578125, + "logps/rejected": -226.7574005126953, + "loss": 0.5813, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.0418616458773613, + "rewards/margins": 0.3720625042915344, + "rewards/rejected": -0.33020082116127014, + "step": 670 + }, + { + "epoch": 0.7, + "learning_rate": 4.255644852659778e-07, + "logits/chosen": -2.0546717643737793, + "logits/rejected": -1.953681230545044, + "logps/chosen": -285.4710388183594, + "logps/rejected": -235.042724609375, + "loss": 0.6163, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": 0.025151943787932396, + "rewards/margins": 0.2663043141365051, + "rewards/rejected": -0.24115240573883057, + "step": 680 + }, + { + "epoch": 0.71, + "learning_rate": 4.236509758897818e-07, + "logits/chosen": -2.008673906326294, + "logits/rejected": -2.008507490158081, + "logps/chosen": -249.8711700439453, + "logps/rejected": -212.5075225830078, + "loss": 0.6118, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": -0.0028725930023938417, + "rewards/margins": 0.3535704016685486, + "rewards/rejected": -0.35644301772117615, + "step": 690 + }, + { + "epoch": 0.72, + "learning_rate": 4.2173746651358586e-07, + "logits/chosen": -2.024738073348999, + "logits/rejected": -1.9645500183105469, + "logps/chosen": -276.9503173828125, + "logps/rejected": -216.16281127929688, + "loss": 0.5986, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": 0.018686672672629356, + "rewards/margins": 0.3331596255302429, + "rewards/rejected": -0.31447291374206543, + "step": 700 + }, + { + "epoch": 0.73, + "learning_rate": 4.198239571373899e-07, + "logits/chosen": -2.060493230819702, + "logits/rejected": -1.9129841327667236, + "logps/chosen": -254.92068481445312, + "logps/rejected": -211.83877563476562, + "loss": 0.5869, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 0.0011024778941646218, + "rewards/margins": 0.3638603985309601, + "rewards/rejected": -0.36275792121887207, + "step": 710 + }, + { + "epoch": 0.74, + "learning_rate": 4.17910447761194e-07, + "logits/chosen": -2.045718193054199, + "logits/rejected": -2.025928497314453, + "logps/chosen": -294.1854248046875, + "logps/rejected": -255.2843017578125, + "loss": 0.5978, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": 0.029152315109968185, + "rewards/margins": 0.2995935380458832, + "rewards/rejected": -0.2704412341117859, + "step": 720 + }, + { + "epoch": 0.75, + "learning_rate": 4.1599693838499805e-07, + "logits/chosen": -2.038595676422119, + "logits/rejected": -1.9576959609985352, + "logps/chosen": -268.7452697753906, + "logps/rejected": -240.6742401123047, + "loss": 0.5823, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": 0.03609790652990341, + "rewards/margins": 0.3287338614463806, + "rewards/rejected": -0.292635977268219, + "step": 730 + }, + { + "epoch": 0.76, + "learning_rate": 4.140834290088021e-07, + "logits/chosen": -2.000624179840088, + "logits/rejected": -1.9431140422821045, + "logps/chosen": -274.5948181152344, + "logps/rejected": -218.09573364257812, + "loss": 0.5771, + "rewards/accuracies": 0.6656249761581421, + "rewards/chosen": -0.0008543640142306685, + "rewards/margins": 0.4075881540775299, + "rewards/rejected": -0.40844249725341797, + "step": 740 + }, + { + "epoch": 0.77, + "learning_rate": 4.121699196326062e-07, + "logits/chosen": -1.9728513956069946, + "logits/rejected": -1.9374927282333374, + "logps/chosen": -255.46408081054688, + "logps/rejected": -212.78280639648438, + "loss": 0.5899, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": 0.01102383527904749, + "rewards/margins": 0.3705114722251892, + "rewards/rejected": -0.3594876229763031, + "step": 750 + }, + { + "epoch": 0.78, + "learning_rate": 4.1025641025641024e-07, + "logits/chosen": -1.9775402545928955, + "logits/rejected": -1.9780277013778687, + "logps/chosen": -252.36978149414062, + "logps/rejected": -251.9305419921875, + "loss": 0.5854, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.045078955590724945, + "rewards/margins": 0.3284622132778168, + "rewards/rejected": -0.3735411763191223, + "step": 760 + }, + { + "epoch": 0.8, + "learning_rate": 4.083429008802143e-07, + "logits/chosen": -2.0461549758911133, + "logits/rejected": -1.897442102432251, + "logps/chosen": -269.2337341308594, + "logps/rejected": -231.2041778564453, + "loss": 0.5673, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 0.01757672056555748, + "rewards/margins": 0.3843061327934265, + "rewards/rejected": -0.3667294383049011, + "step": 770 + }, + { + "epoch": 0.81, + "learning_rate": 4.0642939150401836e-07, + "logits/chosen": -2.1057353019714355, + "logits/rejected": -1.9940143823623657, + "logps/chosen": -267.21270751953125, + "logps/rejected": -229.93746948242188, + "loss": 0.573, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.0010803266195580363, + "rewards/margins": 0.4319356083869934, + "rewards/rejected": -0.433015912771225, + "step": 780 + }, + { + "epoch": 0.82, + "learning_rate": 4.0451588212782237e-07, + "logits/chosen": -2.0213711261749268, + "logits/rejected": -1.9726299047470093, + "logps/chosen": -266.59857177734375, + "logps/rejected": -213.43722534179688, + "loss": 0.5461, + "rewards/accuracies": 0.734375, + "rewards/chosen": 0.03156222775578499, + "rewards/margins": 0.4636878967285156, + "rewards/rejected": -0.4321257174015045, + "step": 790 + }, + { + "epoch": 0.83, + "learning_rate": 4.0260237275162643e-07, + "logits/chosen": -1.9949146509170532, + "logits/rejected": -1.9421812295913696, + "logps/chosen": -277.64373779296875, + "logps/rejected": -235.33944702148438, + "loss": 0.5902, + "rewards/accuracies": 0.6468750238418579, + "rewards/chosen": 0.017059288918972015, + "rewards/margins": 0.35789138078689575, + "rewards/rejected": -0.34083208441734314, + "step": 800 + }, + { + "epoch": 0.84, + "learning_rate": 4.006888633754305e-07, + "logits/chosen": -2.038794994354248, + "logits/rejected": -1.943619728088379, + "logps/chosen": -264.1832580566406, + "logps/rejected": -225.75350952148438, + "loss": 0.5496, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": 0.014380586333572865, + "rewards/margins": 0.4638059735298157, + "rewards/rejected": -0.4494253695011139, + "step": 810 + }, + { + "epoch": 0.85, + "learning_rate": 3.9877535399923456e-07, + "logits/chosen": -2.0996642112731934, + "logits/rejected": -1.9559507369995117, + "logps/chosen": -281.420654296875, + "logps/rejected": -241.8292236328125, + "loss": 0.5706, + "rewards/accuracies": 0.671875, + "rewards/chosen": 0.036143362522125244, + "rewards/margins": 0.47268205881118774, + "rewards/rejected": -0.4365386366844177, + "step": 820 + }, + { + "epoch": 0.86, + "learning_rate": 3.968618446230386e-07, + "logits/chosen": -2.040336847305298, + "logits/rejected": -1.9150043725967407, + "logps/chosen": -299.0230712890625, + "logps/rejected": -229.7438201904297, + "loss": 0.5555, + "rewards/accuracies": 0.793749988079071, + "rewards/chosen": 0.0862685814499855, + "rewards/margins": 0.593079686164856, + "rewards/rejected": -0.5068112015724182, + "step": 830 + }, + { + "epoch": 0.87, + "learning_rate": 3.949483352468427e-07, + "logits/chosen": -2.0034563541412354, + "logits/rejected": -1.9005334377288818, + "logps/chosen": -263.16143798828125, + "logps/rejected": -223.8854522705078, + "loss": 0.6149, + "rewards/accuracies": 0.659375011920929, + "rewards/chosen": -0.07270374894142151, + "rewards/margins": 0.34188348054885864, + "rewards/rejected": -0.41458725929260254, + "step": 840 + }, + { + "epoch": 0.88, + "learning_rate": 3.9303482587064674e-07, + "logits/chosen": -2.024224042892456, + "logits/rejected": -1.9794381856918335, + "logps/chosen": -282.8343200683594, + "logps/rejected": -233.40713500976562, + "loss": 0.5746, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.0055493758991360664, + "rewards/margins": 0.4002881944179535, + "rewards/rejected": -0.4058375954627991, + "step": 850 + }, + { + "epoch": 0.89, + "learning_rate": 3.911213164944508e-07, + "logits/chosen": -2.1257824897766113, + "logits/rejected": -2.0204081535339355, + "logps/chosen": -276.70672607421875, + "logps/rejected": -241.56045532226562, + "loss": 0.5721, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.0032120614778250456, + "rewards/margins": 0.45397186279296875, + "rewards/rejected": -0.45718392729759216, + "step": 860 + }, + { + "epoch": 0.9, + "learning_rate": 3.8920780711825487e-07, + "logits/chosen": -1.9964282512664795, + "logits/rejected": -1.908174753189087, + "logps/chosen": -286.13702392578125, + "logps/rejected": -234.1906280517578, + "loss": 0.5772, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.0039267390966415405, + "rewards/margins": 0.4336286187171936, + "rewards/rejected": -0.43755531311035156, + "step": 870 + }, + { + "epoch": 0.91, + "learning_rate": 3.8729429774205893e-07, + "logits/chosen": -2.024934768676758, + "logits/rejected": -1.9702039957046509, + "logps/chosen": -283.3232421875, + "logps/rejected": -234.4091033935547, + "loss": 0.5749, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.012345449067652225, + "rewards/margins": 0.42594093084335327, + "rewards/rejected": -0.43828636407852173, + "step": 880 + }, + { + "epoch": 0.92, + "learning_rate": 3.8538078836586294e-07, + "logits/chosen": -2.00795578956604, + "logits/rejected": -2.0201306343078613, + "logps/chosen": -253.5358123779297, + "logps/rejected": -228.1576385498047, + "loss": 0.5818, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": 0.026046928018331528, + "rewards/margins": 0.48412972688674927, + "rewards/rejected": -0.45808282494544983, + "step": 890 + }, + { + "epoch": 0.93, + "learning_rate": 3.83467278989667e-07, + "logits/chosen": -1.9805667400360107, + "logits/rejected": -1.8971214294433594, + "logps/chosen": -259.73486328125, + "logps/rejected": -205.6741943359375, + "loss": 0.5589, + "rewards/accuracies": 0.734375, + "rewards/chosen": -0.019825313240289688, + "rewards/margins": 0.4857255816459656, + "rewards/rejected": -0.5055509805679321, + "step": 900 + }, + { + "epoch": 0.94, + "learning_rate": 3.8155376961347106e-07, + "logits/chosen": -2.0091850757598877, + "logits/rejected": -1.965311050415039, + "logps/chosen": -243.44900512695312, + "logps/rejected": -203.65777587890625, + "loss": 0.58, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.024844542145729065, + "rewards/margins": 0.46328315138816833, + "rewards/rejected": -0.4881277084350586, + "step": 910 + }, + { + "epoch": 0.95, + "learning_rate": 3.796402602372751e-07, + "logits/chosen": -2.1025612354278564, + "logits/rejected": -2.0319173336029053, + "logps/chosen": -278.753173828125, + "logps/rejected": -242.23977661132812, + "loss": 0.5718, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.0475626215338707, + "rewards/margins": 0.3921951651573181, + "rewards/rejected": -0.4397578239440918, + "step": 920 + }, + { + "epoch": 0.96, + "learning_rate": 3.777267508610792e-07, + "logits/chosen": -1.9871509075164795, + "logits/rejected": -1.9420807361602783, + "logps/chosen": -247.9656982421875, + "logps/rejected": -206.86196899414062, + "loss": 0.5746, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.032572999596595764, + "rewards/margins": 0.4663190245628357, + "rewards/rejected": -0.49889200925827026, + "step": 930 + }, + { + "epoch": 0.97, + "learning_rate": 3.7581324148488325e-07, + "logits/chosen": -2.034703016281128, + "logits/rejected": -1.9743045568466187, + "logps/chosen": -283.9637451171875, + "logps/rejected": -233.59152221679688, + "loss": 0.5768, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": -0.024589989334344864, + "rewards/margins": 0.41736704111099243, + "rewards/rejected": -0.4419569969177246, + "step": 940 + }, + { + "epoch": 0.98, + "learning_rate": 3.738997321086873e-07, + "logits/chosen": -2.0080621242523193, + "logits/rejected": -1.931701421737671, + "logps/chosen": -269.7777404785156, + "logps/rejected": -223.91110229492188, + "loss": 0.5847, + "rewards/accuracies": 0.671875, + "rewards/chosen": -0.037136368453502655, + "rewards/margins": 0.41323599219322205, + "rewards/rejected": -0.4503723084926605, + "step": 950 + }, + { + "epoch": 0.99, + "learning_rate": 3.7198622273249137e-07, + "logits/chosen": -1.994866132736206, + "logits/rejected": -1.9065024852752686, + "logps/chosen": -264.7047119140625, + "logps/rejected": -227.7999725341797, + "loss": 0.5546, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.01111307181417942, + "rewards/margins": 0.4391903281211853, + "rewards/rejected": -0.45030340552330017, + "step": 960 + }, + { + "epoch": 1.0, + "eval_logits/chosen": -1.9394429922103882, + "eval_logits/rejected": -1.8507767915725708, + "eval_logps/chosen": -282.7758483886719, + "eval_logps/rejected": -234.58616638183594, + "eval_loss": 0.567847490310669, + "eval_rewards/accuracies": 0.7053571343421936, + "eval_rewards/chosen": -0.009052435867488384, + "eval_rewards/margins": 0.4890976846218109, + "eval_rewards/rejected": -0.4981500804424286, + "eval_runtime": 874.7806, + "eval_samples_per_second": 2.286, + "eval_steps_per_second": 0.072, + "step": 968 + }, + { + "epoch": 1.0, + "learning_rate": 3.7007271335629544e-07, + "logits/chosen": -2.007869243621826, + "logits/rejected": -1.9336410760879517, + "logps/chosen": -262.53546142578125, + "logps/rejected": -226.2677764892578, + "loss": 0.5609, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.021611416712403297, + "rewards/margins": 0.47191959619522095, + "rewards/rejected": -0.4935310482978821, + "step": 970 + }, + { + "epoch": 1.01, + "learning_rate": 3.681592039800995e-07, + "logits/chosen": -2.0216825008392334, + "logits/rejected": -1.932154655456543, + "logps/chosen": -253.0286102294922, + "logps/rejected": -214.9832000732422, + "loss": 0.5595, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.021348293870687485, + "rewards/margins": 0.5129477977752686, + "rewards/rejected": -0.5342960953712463, + "step": 980 + }, + { + "epoch": 1.02, + "learning_rate": 3.662456946039035e-07, + "logits/chosen": -1.9851185083389282, + "logits/rejected": -1.8990024328231812, + "logps/chosen": -234.0498046875, + "logps/rejected": -218.21554565429688, + "loss": 0.6073, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": -0.08372095972299576, + "rewards/margins": 0.39530184864997864, + "rewards/rejected": -0.4790228009223938, + "step": 990 + }, + { + "epoch": 1.03, + "learning_rate": 3.6433218522770757e-07, + "logits/chosen": -2.0040273666381836, + "logits/rejected": -1.9534374475479126, + "logps/chosen": -261.78216552734375, + "logps/rejected": -232.15872192382812, + "loss": 0.586, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.06383053213357925, + "rewards/margins": 0.39705249667167664, + "rewards/rejected": -0.4608830511569977, + "step": 1000 + }, + { + "epoch": 1.04, + "learning_rate": 3.6241867585151163e-07, + "logits/chosen": -2.057312250137329, + "logits/rejected": -1.9657714366912842, + "logps/chosen": -271.1720886230469, + "logps/rejected": -239.08377075195312, + "loss": 0.5623, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.03590734675526619, + "rewards/margins": 0.46565335988998413, + "rewards/rejected": -0.5015607476234436, + "step": 1010 + }, + { + "epoch": 1.05, + "learning_rate": 3.605051664753157e-07, + "logits/chosen": -2.0540215969085693, + "logits/rejected": -1.9568777084350586, + "logps/chosen": -274.85723876953125, + "logps/rejected": -215.91989135742188, + "loss": 0.5587, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.06657890975475311, + "rewards/margins": 0.5352650880813599, + "rewards/rejected": -0.6018439531326294, + "step": 1020 + }, + { + "epoch": 1.06, + "learning_rate": 3.5859165709911975e-07, + "logits/chosen": -2.0053391456604004, + "logits/rejected": -1.9338204860687256, + "logps/chosen": -280.01971435546875, + "logps/rejected": -240.06552124023438, + "loss": 0.5448, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": 0.02629667893052101, + "rewards/margins": 0.537010133266449, + "rewards/rejected": -0.510713517665863, + "step": 1030 + }, + { + "epoch": 1.07, + "learning_rate": 3.566781477229238e-07, + "logits/chosen": -2.038978099822998, + "logits/rejected": -1.9422070980072021, + "logps/chosen": -271.25006103515625, + "logps/rejected": -247.28543090820312, + "loss": 0.5631, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": 0.002108191605657339, + "rewards/margins": 0.47303467988967896, + "rewards/rejected": -0.4709264636039734, + "step": 1040 + }, + { + "epoch": 1.08, + "learning_rate": 3.547646383467279e-07, + "logits/chosen": -2.0076701641082764, + "logits/rejected": -1.9359521865844727, + "logps/chosen": -265.30242919921875, + "logps/rejected": -219.84677124023438, + "loss": 0.5406, + "rewards/accuracies": 0.753125011920929, + "rewards/chosen": -0.022836418822407722, + "rewards/margins": 0.5607357621192932, + "rewards/rejected": -0.5835721492767334, + "step": 1050 + }, + { + "epoch": 1.09, + "learning_rate": 3.5285112897053194e-07, + "logits/chosen": -2.00288462638855, + "logits/rejected": -1.867516279220581, + "logps/chosen": -261.2705993652344, + "logps/rejected": -231.8849334716797, + "loss": 0.5549, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.03538358584046364, + "rewards/margins": 0.5409261584281921, + "rewards/rejected": -0.5763096809387207, + "step": 1060 + }, + { + "epoch": 1.11, + "learning_rate": 3.50937619594336e-07, + "logits/chosen": -2.028400421142578, + "logits/rejected": -1.8853849172592163, + "logps/chosen": -275.1874084472656, + "logps/rejected": -231.4445037841797, + "loss": 0.5522, + "rewards/accuracies": 0.765625, + "rewards/chosen": -0.019299419596791267, + "rewards/margins": 0.51097571849823, + "rewards/rejected": -0.5302751660346985, + "step": 1070 + }, + { + "epoch": 1.12, + "learning_rate": 3.4902411021814007e-07, + "logits/chosen": -2.072242498397827, + "logits/rejected": -1.9163639545440674, + "logps/chosen": -270.66961669921875, + "logps/rejected": -220.4494171142578, + "loss": 0.5427, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.027892639860510826, + "rewards/margins": 0.49505919218063354, + "rewards/rejected": -0.5229519009590149, + "step": 1080 + }, + { + "epoch": 1.13, + "learning_rate": 3.4711060084194413e-07, + "logits/chosen": -1.9672114849090576, + "logits/rejected": -1.8720566034317017, + "logps/chosen": -266.3045959472656, + "logps/rejected": -223.64028930664062, + "loss": 0.5794, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.08450685441493988, + "rewards/margins": 0.42633262276649475, + "rewards/rejected": -0.510839581489563, + "step": 1090 + }, + { + "epoch": 1.14, + "learning_rate": 3.4519709146574814e-07, + "logits/chosen": -1.9523357152938843, + "logits/rejected": -1.9406931400299072, + "logps/chosen": -243.3000946044922, + "logps/rejected": -213.4463348388672, + "loss": 0.5655, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.11162979900836945, + "rewards/margins": 0.41768351197242737, + "rewards/rejected": -0.5293132662773132, + "step": 1100 + }, + { + "epoch": 1.15, + "learning_rate": 3.432835820895522e-07, + "logits/chosen": -2.0237271785736084, + "logits/rejected": -1.9257009029388428, + "logps/chosen": -268.1784362792969, + "logps/rejected": -244.5179443359375, + "loss": 0.5485, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.0015948250656947494, + "rewards/margins": 0.519138514995575, + "rewards/rejected": -0.5175436735153198, + "step": 1110 + }, + { + "epoch": 1.16, + "learning_rate": 3.4137007271335626e-07, + "logits/chosen": -1.979138970375061, + "logits/rejected": -1.8770424127578735, + "logps/chosen": -248.8733673095703, + "logps/rejected": -217.2586669921875, + "loss": 0.5551, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.07530103623867035, + "rewards/margins": 0.49078720808029175, + "rewards/rejected": -0.5660881996154785, + "step": 1120 + }, + { + "epoch": 1.17, + "learning_rate": 3.394565633371603e-07, + "logits/chosen": -1.9930393695831299, + "logits/rejected": -1.8950424194335938, + "logps/chosen": -284.9536437988281, + "logps/rejected": -229.8956298828125, + "loss": 0.5479, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.04036648944020271, + "rewards/margins": 0.5784170627593994, + "rewards/rejected": -0.6187835931777954, + "step": 1130 + }, + { + "epoch": 1.18, + "learning_rate": 3.375430539609644e-07, + "logits/chosen": -1.9363939762115479, + "logits/rejected": -1.8576284646987915, + "logps/chosen": -259.5916748046875, + "logps/rejected": -230.9806365966797, + "loss": 0.524, + "rewards/accuracies": 0.734375, + "rewards/chosen": -0.06810829788446426, + "rewards/margins": 0.5644280314445496, + "rewards/rejected": -0.6325362920761108, + "step": 1140 + }, + { + "epoch": 1.19, + "learning_rate": 3.3562954458476845e-07, + "logits/chosen": -2.0367255210876465, + "logits/rejected": -1.9168630838394165, + "logps/chosen": -275.58404541015625, + "logps/rejected": -228.10073852539062, + "loss": 0.5655, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.10140882432460785, + "rewards/margins": 0.46295589208602905, + "rewards/rejected": -0.5643647909164429, + "step": 1150 + }, + { + "epoch": 1.2, + "learning_rate": 3.337160352085725e-07, + "logits/chosen": -1.9923851490020752, + "logits/rejected": -1.8931716680526733, + "logps/chosen": -288.2662048339844, + "logps/rejected": -243.01809692382812, + "loss": 0.5404, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.05664486810564995, + "rewards/margins": 0.5657026171684265, + "rewards/rejected": -0.6223475337028503, + "step": 1160 + }, + { + "epoch": 1.21, + "learning_rate": 3.3180252583237657e-07, + "logits/chosen": -2.015646457672119, + "logits/rejected": -1.8909307718276978, + "logps/chosen": -284.00213623046875, + "logps/rejected": -226.9560089111328, + "loss": 0.5686, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.03462731093168259, + "rewards/margins": 0.5479375123977661, + "rewards/rejected": -0.5825648903846741, + "step": 1170 + }, + { + "epoch": 1.22, + "learning_rate": 3.2988901645618063e-07, + "logits/chosen": -1.993431806564331, + "logits/rejected": -1.8949865102767944, + "logps/chosen": -274.4952392578125, + "logps/rejected": -233.14193725585938, + "loss": 0.5515, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.11015860736370087, + "rewards/margins": 0.4747910499572754, + "rewards/rejected": -0.5849496722221375, + "step": 1180 + }, + { + "epoch": 1.23, + "learning_rate": 3.279755070799847e-07, + "logits/chosen": -1.9973537921905518, + "logits/rejected": -1.937077283859253, + "logps/chosen": -267.6756896972656, + "logps/rejected": -239.6106414794922, + "loss": 0.5382, + "rewards/accuracies": 0.7718750238418579, + "rewards/chosen": -0.05113865062594414, + "rewards/margins": 0.5464734435081482, + "rewards/rejected": -0.5976120829582214, + "step": 1190 + }, + { + "epoch": 1.24, + "learning_rate": 3.260619977037887e-07, + "logits/chosen": -1.9573997259140015, + "logits/rejected": -1.861267328262329, + "logps/chosen": -258.67901611328125, + "logps/rejected": -224.18716430664062, + "loss": 0.5347, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.07424889504909515, + "rewards/margins": 0.5720986723899841, + "rewards/rejected": -0.6463476419448853, + "step": 1200 + }, + { + "epoch": 1.25, + "learning_rate": 3.2414848832759277e-07, + "logits/chosen": -2.055069923400879, + "logits/rejected": -1.9554643630981445, + "logps/chosen": -266.70721435546875, + "logps/rejected": -230.57394409179688, + "loss": 0.536, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.05199545621871948, + "rewards/margins": 0.5812665224075317, + "rewards/rejected": -0.6332619190216064, + "step": 1210 + }, + { + "epoch": 1.26, + "learning_rate": 3.2223497895139683e-07, + "logits/chosen": -2.010925769805908, + "logits/rejected": -1.9447513818740845, + "logps/chosen": -263.6624755859375, + "logps/rejected": -239.1702423095703, + "loss": 0.5616, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.0350295789539814, + "rewards/margins": 0.4964603781700134, + "rewards/rejected": -0.5314900279045105, + "step": 1220 + }, + { + "epoch": 1.27, + "learning_rate": 3.203214695752009e-07, + "logits/chosen": -2.0407071113586426, + "logits/rejected": -1.9032310247421265, + "logps/chosen": -290.69842529296875, + "logps/rejected": -240.09481811523438, + "loss": 0.575, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.08812502771615982, + "rewards/margins": 0.4931270182132721, + "rewards/rejected": -0.5812520384788513, + "step": 1230 + }, + { + "epoch": 1.28, + "learning_rate": 3.1840796019900495e-07, + "logits/chosen": -2.0096232891082764, + "logits/rejected": -1.9704147577285767, + "logps/chosen": -275.1475830078125, + "logps/rejected": -246.9510040283203, + "loss": 0.5631, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.024968769401311874, + "rewards/margins": 0.5439773797988892, + "rewards/rejected": -0.5689461827278137, + "step": 1240 + }, + { + "epoch": 1.29, + "learning_rate": 3.16494450822809e-07, + "logits/chosen": -2.008104085922241, + "logits/rejected": -1.9298263788223267, + "logps/chosen": -273.0186767578125, + "logps/rejected": -211.26467895507812, + "loss": 0.5398, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.08886951208114624, + "rewards/margins": 0.5550605654716492, + "rewards/rejected": -0.6439300775527954, + "step": 1250 + }, + { + "epoch": 1.3, + "learning_rate": 3.145809414466131e-07, + "logits/chosen": -2.024360418319702, + "logits/rejected": -1.9324896335601807, + "logps/chosen": -269.1455078125, + "logps/rejected": -233.44287109375, + "loss": 0.5557, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.016493279486894608, + "rewards/margins": 0.541469931602478, + "rewards/rejected": -0.5579632520675659, + "step": 1260 + }, + { + "epoch": 1.31, + "learning_rate": 3.1266743207041714e-07, + "logits/chosen": -1.9940264225006104, + "logits/rejected": -1.9849786758422852, + "logps/chosen": -279.35693359375, + "logps/rejected": -226.4078369140625, + "loss": 0.5579, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.09030825644731522, + "rewards/margins": 0.5069442391395569, + "rewards/rejected": -0.5972524881362915, + "step": 1270 + }, + { + "epoch": 1.32, + "learning_rate": 3.107539226942212e-07, + "logits/chosen": -1.980614423751831, + "logits/rejected": -1.9127609729766846, + "logps/chosen": -258.0218200683594, + "logps/rejected": -236.0473175048828, + "loss": 0.5912, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.06303338706493378, + "rewards/margins": 0.47944989800453186, + "rewards/rejected": -0.5424833297729492, + "step": 1280 + }, + { + "epoch": 1.33, + "learning_rate": 3.0884041331802526e-07, + "logits/chosen": -2.0076379776000977, + "logits/rejected": -1.9233248233795166, + "logps/chosen": -267.875244140625, + "logps/rejected": -229.68124389648438, + "loss": 0.5583, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.08163460344076157, + "rewards/margins": 0.5768612027168274, + "rewards/rejected": -0.6584957242012024, + "step": 1290 + }, + { + "epoch": 1.34, + "learning_rate": 3.0692690394182927e-07, + "logits/chosen": -2.0399606227874756, + "logits/rejected": -2.0132060050964355, + "logps/chosen": -290.6252746582031, + "logps/rejected": -235.99917602539062, + "loss": 0.5551, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.018822865560650826, + "rewards/margins": 0.5866366624832153, + "rewards/rejected": -0.6054595112800598, + "step": 1300 + }, + { + "epoch": 1.35, + "learning_rate": 3.0501339456563334e-07, + "logits/chosen": -2.0360805988311768, + "logits/rejected": -1.9700591564178467, + "logps/chosen": -269.5166015625, + "logps/rejected": -219.4673309326172, + "loss": 0.5436, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.040003370493650436, + "rewards/margins": 0.5832095742225647, + "rewards/rejected": -0.6232129335403442, + "step": 1310 + }, + { + "epoch": 1.36, + "learning_rate": 3.030998851894374e-07, + "logits/chosen": -1.9755254983901978, + "logits/rejected": -1.872722864151001, + "logps/chosen": -236.5614471435547, + "logps/rejected": -203.92333984375, + "loss": 0.5274, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.09586571156978607, + "rewards/margins": 0.5836740732192993, + "rewards/rejected": -0.6795397996902466, + "step": 1320 + }, + { + "epoch": 1.37, + "learning_rate": 3.0118637581324146e-07, + "logits/chosen": -1.9765536785125732, + "logits/rejected": -1.9652169942855835, + "logps/chosen": -251.5444793701172, + "logps/rejected": -222.391357421875, + "loss": 0.5565, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.10412011295557022, + "rewards/margins": 0.5656259655952454, + "rewards/rejected": -0.6697460412979126, + "step": 1330 + }, + { + "epoch": 1.38, + "learning_rate": 2.992728664370455e-07, + "logits/chosen": -1.9934202432632446, + "logits/rejected": -1.918143630027771, + "logps/chosen": -251.29257202148438, + "logps/rejected": -215.9584197998047, + "loss": 0.5577, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.0995405912399292, + "rewards/margins": 0.5579150915145874, + "rewards/rejected": -0.657455563545227, + "step": 1340 + }, + { + "epoch": 1.39, + "learning_rate": 2.973593570608496e-07, + "logits/chosen": -1.9801356792449951, + "logits/rejected": -1.8740001916885376, + "logps/chosen": -269.63409423828125, + "logps/rejected": -226.4136505126953, + "loss": 0.5604, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.07889227569103241, + "rewards/margins": 0.6068841218948364, + "rewards/rejected": -0.68577641248703, + "step": 1350 + }, + { + "epoch": 1.4, + "learning_rate": 2.9544584768465365e-07, + "logits/chosen": -1.9733374118804932, + "logits/rejected": -1.8860597610473633, + "logps/chosen": -270.7989196777344, + "logps/rejected": -233.51712036132812, + "loss": 0.5682, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": -0.12125426530838013, + "rewards/margins": 0.4999443590641022, + "rewards/rejected": -0.6211986541748047, + "step": 1360 + }, + { + "epoch": 1.41, + "learning_rate": 2.935323383084577e-07, + "logits/chosen": -2.055373191833496, + "logits/rejected": -1.9579395055770874, + "logps/chosen": -284.0872497558594, + "logps/rejected": -218.99630737304688, + "loss": 0.5564, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.09817256033420563, + "rewards/margins": 0.6084792613983154, + "rewards/rejected": -0.7066518068313599, + "step": 1370 + }, + { + "epoch": 1.43, + "learning_rate": 2.9161882893226177e-07, + "logits/chosen": -1.9930992126464844, + "logits/rejected": -1.8951447010040283, + "logps/chosen": -263.5531005859375, + "logps/rejected": -220.1865692138672, + "loss": 0.5546, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.08504018932580948, + "rewards/margins": 0.5876429080963135, + "rewards/rejected": -0.67268306016922, + "step": 1380 + }, + { + "epoch": 1.44, + "learning_rate": 2.8970531955606583e-07, + "logits/chosen": -2.061654806137085, + "logits/rejected": -1.9857797622680664, + "logps/chosen": -281.9578552246094, + "logps/rejected": -235.69680786132812, + "loss": 0.5682, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.08588355034589767, + "rewards/margins": 0.5460136532783508, + "rewards/rejected": -0.6318972706794739, + "step": 1390 + }, + { + "epoch": 1.45, + "learning_rate": 2.8779181017986984e-07, + "logits/chosen": -2.0682482719421387, + "logits/rejected": -1.935006856918335, + "logps/chosen": -277.1342468261719, + "logps/rejected": -245.4555206298828, + "loss": 0.5532, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.0687127336859703, + "rewards/margins": 0.5643824934959412, + "rewards/rejected": -0.6330951452255249, + "step": 1400 + }, + { + "epoch": 1.46, + "learning_rate": 2.858783008036739e-07, + "logits/chosen": -1.976893663406372, + "logits/rejected": -1.8818718194961548, + "logps/chosen": -265.47113037109375, + "logps/rejected": -224.09805297851562, + "loss": 0.545, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.10257352888584137, + "rewards/margins": 0.539416491985321, + "rewards/rejected": -0.641990065574646, + "step": 1410 + }, + { + "epoch": 1.47, + "learning_rate": 2.8396479142747797e-07, + "logits/chosen": -2.019167184829712, + "logits/rejected": -1.9711027145385742, + "logps/chosen": -269.1687927246094, + "logps/rejected": -226.229736328125, + "loss": 0.5455, + "rewards/accuracies": 0.746874988079071, + "rewards/chosen": -0.07154490053653717, + "rewards/margins": 0.6214567422866821, + "rewards/rejected": -0.6930015683174133, + "step": 1420 + }, + { + "epoch": 1.48, + "learning_rate": 2.8205128205128203e-07, + "logits/chosen": -1.9249579906463623, + "logits/rejected": -1.9243910312652588, + "logps/chosen": -255.308837890625, + "logps/rejected": -243.5002899169922, + "loss": 0.5503, + "rewards/accuracies": 0.671875, + "rewards/chosen": -0.08084328472614288, + "rewards/margins": 0.5392208099365234, + "rewards/rejected": -0.6200639605522156, + "step": 1430 + }, + { + "epoch": 1.49, + "learning_rate": 2.801377726750861e-07, + "logits/chosen": -2.0300347805023193, + "logits/rejected": -1.9318290948867798, + "logps/chosen": -263.9398498535156, + "logps/rejected": -231.84042358398438, + "loss": 0.5516, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.048500318080186844, + "rewards/margins": 0.5793226957321167, + "rewards/rejected": -0.6278230547904968, + "step": 1440 + }, + { + "epoch": 1.5, + "learning_rate": 2.7822426329889015e-07, + "logits/chosen": -1.9659664630889893, + "logits/rejected": -1.8714187145233154, + "logps/chosen": -249.6382293701172, + "logps/rejected": -217.23147583007812, + "loss": 0.578, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.14306652545928955, + "rewards/margins": 0.5336703062057495, + "rewards/rejected": -0.6767368316650391, + "step": 1450 + }, + { + "epoch": 1.51, + "learning_rate": 2.763107539226942e-07, + "logits/chosen": -2.0092709064483643, + "logits/rejected": -1.8980295658111572, + "logps/chosen": -263.8819885253906, + "logps/rejected": -228.8859100341797, + "loss": 0.563, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.11218663305044174, + "rewards/margins": 0.49865245819091797, + "rewards/rejected": -0.6108390688896179, + "step": 1460 + }, + { + "epoch": 1.52, + "learning_rate": 2.743972445464983e-07, + "logits/chosen": -1.9666690826416016, + "logits/rejected": -1.9279935359954834, + "logps/chosen": -260.71295166015625, + "logps/rejected": -216.5955047607422, + "loss": 0.5668, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.12362690269947052, + "rewards/margins": 0.5188626050949097, + "rewards/rejected": -0.642489492893219, + "step": 1470 + }, + { + "epoch": 1.53, + "learning_rate": 2.7248373517030234e-07, + "logits/chosen": -2.0014965534210205, + "logits/rejected": -1.8935635089874268, + "logps/chosen": -254.9728546142578, + "logps/rejected": -228.10171508789062, + "loss": 0.5456, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": -0.1154770627617836, + "rewards/margins": 0.5314809083938599, + "rewards/rejected": -0.6469579935073853, + "step": 1480 + }, + { + "epoch": 1.54, + "learning_rate": 2.705702257941064e-07, + "logits/chosen": -2.0562753677368164, + "logits/rejected": -1.9150091409683228, + "logps/chosen": -304.14825439453125, + "logps/rejected": -234.85153198242188, + "loss": 0.5296, + "rewards/accuracies": 0.778124988079071, + "rewards/chosen": -0.06085296720266342, + "rewards/margins": 0.620803952217102, + "rewards/rejected": -0.6816568374633789, + "step": 1490 + }, + { + "epoch": 1.55, + "learning_rate": 2.686567164179104e-07, + "logits/chosen": -2.017127513885498, + "logits/rejected": -1.9755398035049438, + "logps/chosen": -258.0429992675781, + "logps/rejected": -211.6239776611328, + "loss": 0.5573, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.11576205492019653, + "rewards/margins": 0.5504833459854126, + "rewards/rejected": -0.6662454009056091, + "step": 1500 + }, + { + "epoch": 1.56, + "learning_rate": 2.6674320704171447e-07, + "logits/chosen": -1.97178053855896, + "logits/rejected": -1.9140660762786865, + "logps/chosen": -265.73797607421875, + "logps/rejected": -215.9640655517578, + "loss": 0.532, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07512752711772919, + "rewards/margins": 0.6296411752700806, + "rewards/rejected": -0.7047686576843262, + "step": 1510 + }, + { + "epoch": 1.57, + "learning_rate": 2.6482969766551853e-07, + "logits/chosen": -1.9850788116455078, + "logits/rejected": -1.9302679300308228, + "logps/chosen": -242.3767547607422, + "logps/rejected": -230.31936645507812, + "loss": 0.5533, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.0870606005191803, + "rewards/margins": 0.502352774143219, + "rewards/rejected": -0.5894134044647217, + "step": 1520 + }, + { + "epoch": 1.58, + "learning_rate": 2.629161882893226e-07, + "logits/chosen": -1.9834896326065063, + "logits/rejected": -1.944684624671936, + "logps/chosen": -276.28167724609375, + "logps/rejected": -224.51611328125, + "loss": 0.5256, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.08709590137004852, + "rewards/margins": 0.6131010055541992, + "rewards/rejected": -0.7001968622207642, + "step": 1530 + }, + { + "epoch": 1.59, + "learning_rate": 2.6100267891312666e-07, + "logits/chosen": -2.0222060680389404, + "logits/rejected": -1.9368455410003662, + "logps/chosen": -283.56488037109375, + "logps/rejected": -226.9105224609375, + "loss": 0.5702, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.0962596982717514, + "rewards/margins": 0.5741952061653137, + "rewards/rejected": -0.6704548597335815, + "step": 1540 + }, + { + "epoch": 1.6, + "learning_rate": 2.590891695369307e-07, + "logits/chosen": -1.9781615734100342, + "logits/rejected": -1.8281803131103516, + "logps/chosen": -259.8866271972656, + "logps/rejected": -207.38735961914062, + "loss": 0.5389, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.04752311855554581, + "rewards/margins": 0.6475936770439148, + "rewards/rejected": -0.6951168179512024, + "step": 1550 + }, + { + "epoch": 1.61, + "learning_rate": 2.571756601607348e-07, + "logits/chosen": -2.0197582244873047, + "logits/rejected": -1.9641326665878296, + "logps/chosen": -285.61285400390625, + "logps/rejected": -232.82080078125, + "loss": 0.5437, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.07258148491382599, + "rewards/margins": 0.6079379320144653, + "rewards/rejected": -0.6805194616317749, + "step": 1560 + }, + { + "epoch": 1.62, + "learning_rate": 2.5526215078453884e-07, + "logits/chosen": -2.0040359497070312, + "logits/rejected": -2.0012545585632324, + "logps/chosen": -277.3008728027344, + "logps/rejected": -250.6426544189453, + "loss": 0.5448, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.08066655695438385, + "rewards/margins": 0.5909748077392578, + "rewards/rejected": -0.6716413497924805, + "step": 1570 + }, + { + "epoch": 1.63, + "learning_rate": 2.533486414083429e-07, + "logits/chosen": -2.0492279529571533, + "logits/rejected": -2.0106427669525146, + "logps/chosen": -290.88690185546875, + "logps/rejected": -222.43844604492188, + "loss": 0.5248, + "rewards/accuracies": 0.7718750238418579, + "rewards/chosen": -0.05663519352674484, + "rewards/margins": 0.7258026003837585, + "rewards/rejected": -0.782437801361084, + "step": 1580 + }, + { + "epoch": 1.64, + "learning_rate": 2.5143513203214697e-07, + "logits/chosen": -2.0188114643096924, + "logits/rejected": -1.9185588359832764, + "logps/chosen": -271.60699462890625, + "logps/rejected": -232.2000732421875, + "loss": 0.5615, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.07878180593252182, + "rewards/margins": 0.5395839810371399, + "rewards/rejected": -0.6183658242225647, + "step": 1590 + }, + { + "epoch": 1.65, + "learning_rate": 2.49521622655951e-07, + "logits/chosen": -1.9827849864959717, + "logits/rejected": -1.8625599145889282, + "logps/chosen": -269.30218505859375, + "logps/rejected": -236.34426879882812, + "loss": 0.5602, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.09624549746513367, + "rewards/margins": 0.5341801047325134, + "rewards/rejected": -0.6304256319999695, + "step": 1600 + }, + { + "epoch": 1.66, + "learning_rate": 2.4760811327975504e-07, + "logits/chosen": -2.0221943855285645, + "logits/rejected": -1.9246753454208374, + "logps/chosen": -268.66131591796875, + "logps/rejected": -226.4938507080078, + "loss": 0.5467, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.11650047451257706, + "rewards/margins": 0.5621265172958374, + "rewards/rejected": -0.6786269545555115, + "step": 1610 + }, + { + "epoch": 1.67, + "learning_rate": 2.456946039035591e-07, + "logits/chosen": -1.9986492395401, + "logits/rejected": -2.009213924407959, + "logps/chosen": -278.3379211425781, + "logps/rejected": -229.5263214111328, + "loss": 0.5695, + "rewards/accuracies": 0.6656249761581421, + "rewards/chosen": -0.15835532546043396, + "rewards/margins": 0.4587886929512024, + "rewards/rejected": -0.617143988609314, + "step": 1620 + }, + { + "epoch": 1.68, + "learning_rate": 2.4378109452736316e-07, + "logits/chosen": -2.032945156097412, + "logits/rejected": -1.9581798315048218, + "logps/chosen": -275.6993103027344, + "logps/rejected": -223.4126739501953, + "loss": 0.5353, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.030741268768906593, + "rewards/margins": 0.6569101810455322, + "rewards/rejected": -0.6876514554023743, + "step": 1630 + }, + { + "epoch": 1.69, + "learning_rate": 2.418675851511672e-07, + "logits/chosen": -1.9015934467315674, + "logits/rejected": -1.951825737953186, + "logps/chosen": -260.4790954589844, + "logps/rejected": -224.67916870117188, + "loss": 0.5695, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.10039174556732178, + "rewards/margins": 0.5715700387954712, + "rewards/rejected": -0.6719617247581482, + "step": 1640 + }, + { + "epoch": 1.7, + "learning_rate": 2.399540757749713e-07, + "logits/chosen": -2.040576934814453, + "logits/rejected": -1.9723045825958252, + "logps/chosen": -276.28387451171875, + "logps/rejected": -231.3341064453125, + "loss": 0.5466, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.10415879637002945, + "rewards/margins": 0.5966567993164062, + "rewards/rejected": -0.7008155584335327, + "step": 1650 + }, + { + "epoch": 1.71, + "learning_rate": 2.3804056639877535e-07, + "logits/chosen": -1.9606773853302002, + "logits/rejected": -1.8021656274795532, + "logps/chosen": -257.7226257324219, + "logps/rejected": -222.80703735351562, + "loss": 0.564, + "rewards/accuracies": 0.746874988079071, + "rewards/chosen": -0.02898462489247322, + "rewards/margins": 0.6081128120422363, + "rewards/rejected": -0.6370974779129028, + "step": 1660 + }, + { + "epoch": 1.72, + "learning_rate": 2.361270570225794e-07, + "logits/chosen": -1.965126633644104, + "logits/rejected": -1.925889015197754, + "logps/chosen": -280.3318176269531, + "logps/rejected": -220.78652954101562, + "loss": 0.521, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.013979366049170494, + "rewards/margins": 0.6451238393783569, + "rewards/rejected": -0.6591032147407532, + "step": 1670 + }, + { + "epoch": 1.74, + "learning_rate": 2.3421354764638345e-07, + "logits/chosen": -2.068727731704712, + "logits/rejected": -1.9082515239715576, + "logps/chosen": -275.98199462890625, + "logps/rejected": -241.04684448242188, + "loss": 0.564, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.09447016566991806, + "rewards/margins": 0.5412524342536926, + "rewards/rejected": -0.6357226371765137, + "step": 1680 + }, + { + "epoch": 1.75, + "learning_rate": 2.323000382701875e-07, + "logits/chosen": -2.0002846717834473, + "logits/rejected": -1.9545902013778687, + "logps/chosen": -280.78656005859375, + "logps/rejected": -230.1374969482422, + "loss": 0.5251, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.06173492223024368, + "rewards/margins": 0.6311557292938232, + "rewards/rejected": -0.6928905844688416, + "step": 1690 + }, + { + "epoch": 1.76, + "learning_rate": 2.3038652889399157e-07, + "logits/chosen": -1.9079262018203735, + "logits/rejected": -1.8384666442871094, + "logps/chosen": -243.6171112060547, + "logps/rejected": -227.16397094726562, + "loss": 0.531, + "rewards/accuracies": 0.746874988079071, + "rewards/chosen": -0.12050791084766388, + "rewards/margins": 0.5977984666824341, + "rewards/rejected": -0.7183063626289368, + "step": 1700 + }, + { + "epoch": 1.77, + "learning_rate": 2.2847301951779563e-07, + "logits/chosen": -1.978109359741211, + "logits/rejected": -1.9294769763946533, + "logps/chosen": -262.1457824707031, + "logps/rejected": -238.79464721679688, + "loss": 0.5353, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.12331397831439972, + "rewards/margins": 0.6272130608558655, + "rewards/rejected": -0.750527024269104, + "step": 1710 + }, + { + "epoch": 1.78, + "learning_rate": 2.265595101415997e-07, + "logits/chosen": -1.9727439880371094, + "logits/rejected": -1.8363845348358154, + "logps/chosen": -278.3986511230469, + "logps/rejected": -249.4431610107422, + "loss": 0.5496, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.06688477098941803, + "rewards/margins": 0.6391483545303345, + "rewards/rejected": -0.7060332298278809, + "step": 1720 + }, + { + "epoch": 1.79, + "learning_rate": 2.2464600076540373e-07, + "logits/chosen": -2.069528102874756, + "logits/rejected": -2.0197949409484863, + "logps/chosen": -289.6525573730469, + "logps/rejected": -228.1117401123047, + "loss": 0.5426, + "rewards/accuracies": 0.746874988079071, + "rewards/chosen": -0.09579763561487198, + "rewards/margins": 0.6648635864257812, + "rewards/rejected": -0.7606611847877502, + "step": 1730 + }, + { + "epoch": 1.8, + "learning_rate": 2.227324913892078e-07, + "logits/chosen": -1.9910539388656616, + "logits/rejected": -1.8664817810058594, + "logps/chosen": -275.5558166503906, + "logps/rejected": -226.4915771484375, + "loss": 0.5386, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.14371807873249054, + "rewards/margins": 0.6043248176574707, + "rewards/rejected": -0.7480429410934448, + "step": 1740 + }, + { + "epoch": 1.81, + "learning_rate": 2.2081898201301186e-07, + "logits/chosen": -2.0246949195861816, + "logits/rejected": -1.9563093185424805, + "logps/chosen": -275.5317077636719, + "logps/rejected": -229.1376190185547, + "loss": 0.5386, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.08551143109798431, + "rewards/margins": 0.5989985466003418, + "rewards/rejected": -0.6845099329948425, + "step": 1750 + }, + { + "epoch": 1.82, + "learning_rate": 2.1890547263681592e-07, + "logits/chosen": -1.9884989261627197, + "logits/rejected": -1.9024312496185303, + "logps/chosen": -263.49151611328125, + "logps/rejected": -228.72201538085938, + "loss": 0.5394, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.1202956885099411, + "rewards/margins": 0.6059435606002808, + "rewards/rejected": -0.7262393236160278, + "step": 1760 + }, + { + "epoch": 1.83, + "learning_rate": 2.1699196326061998e-07, + "logits/chosen": -2.0003156661987305, + "logits/rejected": -1.931420922279358, + "logps/chosen": -247.48104858398438, + "logps/rejected": -230.5535430908203, + "loss": 0.5538, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.18307314813137054, + "rewards/margins": 0.5675541162490845, + "rewards/rejected": -0.7506272196769714, + "step": 1770 + }, + { + "epoch": 1.84, + "learning_rate": 2.1507845388442402e-07, + "logits/chosen": -2.006080150604248, + "logits/rejected": -1.950289011001587, + "logps/chosen": -259.63653564453125, + "logps/rejected": -214.5558624267578, + "loss": 0.5366, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.09678243100643158, + "rewards/margins": 0.556891918182373, + "rewards/rejected": -0.6536744236946106, + "step": 1780 + }, + { + "epoch": 1.85, + "learning_rate": 2.1316494450822808e-07, + "logits/chosen": -2.001936912536621, + "logits/rejected": -1.9808794260025024, + "logps/chosen": -274.71990966796875, + "logps/rejected": -227.1686553955078, + "loss": 0.5538, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.14255772531032562, + "rewards/margins": 0.6057079434394836, + "rewards/rejected": -0.7482656836509705, + "step": 1790 + }, + { + "epoch": 1.86, + "learning_rate": 2.1125143513203214e-07, + "logits/chosen": -1.9766113758087158, + "logits/rejected": -1.8938961029052734, + "logps/chosen": -275.33319091796875, + "logps/rejected": -217.985595703125, + "loss": 0.5274, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.05210094526410103, + "rewards/margins": 0.6903704404830933, + "rewards/rejected": -0.7424713969230652, + "step": 1800 + }, + { + "epoch": 1.87, + "learning_rate": 2.093379257558362e-07, + "logits/chosen": -2.008404493331909, + "logits/rejected": -1.8764822483062744, + "logps/chosen": -264.9756774902344, + "logps/rejected": -230.31790161132812, + "loss": 0.5565, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.1377423107624054, + "rewards/margins": 0.545425295829773, + "rewards/rejected": -0.6831676363945007, + "step": 1810 + }, + { + "epoch": 1.88, + "learning_rate": 2.0742441637964026e-07, + "logits/chosen": -1.948041319847107, + "logits/rejected": -1.9308229684829712, + "logps/chosen": -286.62335205078125, + "logps/rejected": -231.4746856689453, + "loss": 0.5245, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.09207528829574585, + "rewards/margins": 0.6919302940368652, + "rewards/rejected": -0.7840056419372559, + "step": 1820 + }, + { + "epoch": 1.89, + "learning_rate": 2.055109070034443e-07, + "logits/chosen": -2.0039772987365723, + "logits/rejected": -1.9030472040176392, + "logps/chosen": -279.74847412109375, + "logps/rejected": -234.2050323486328, + "loss": 0.5282, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.15545189380645752, + "rewards/margins": 0.535307765007019, + "rewards/rejected": -0.6907596588134766, + "step": 1830 + }, + { + "epoch": 1.9, + "learning_rate": 2.0359739762724836e-07, + "logits/chosen": -2.042912006378174, + "logits/rejected": -1.9346129894256592, + "logps/chosen": -291.7068786621094, + "logps/rejected": -240.38339233398438, + "loss": 0.5332, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.10581548511981964, + "rewards/margins": 0.67552649974823, + "rewards/rejected": -0.7813419699668884, + "step": 1840 + }, + { + "epoch": 1.91, + "learning_rate": 2.0168388825105242e-07, + "logits/chosen": -2.014024257659912, + "logits/rejected": -1.8882499933242798, + "logps/chosen": -274.66912841796875, + "logps/rejected": -245.4554443359375, + "loss": 0.5265, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.06417706608772278, + "rewards/margins": 0.7017583847045898, + "rewards/rejected": -0.7659355401992798, + "step": 1850 + }, + { + "epoch": 1.92, + "learning_rate": 1.997703788748565e-07, + "logits/chosen": -2.037475109100342, + "logits/rejected": -1.9030876159667969, + "logps/chosen": -286.89794921875, + "logps/rejected": -218.78182983398438, + "loss": 0.5452, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.0933770015835762, + "rewards/margins": 0.6915080547332764, + "rewards/rejected": -0.7848849296569824, + "step": 1860 + }, + { + "epoch": 1.93, + "learning_rate": 1.9785686949866055e-07, + "logits/chosen": -1.995123267173767, + "logits/rejected": -1.9561309814453125, + "logps/chosen": -275.4853515625, + "logps/rejected": -236.72006225585938, + "loss": 0.5678, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.15089653432369232, + "rewards/margins": 0.46189427375793457, + "rewards/rejected": -0.6127907633781433, + "step": 1870 + }, + { + "epoch": 1.94, + "learning_rate": 1.9594336012246458e-07, + "logits/chosen": -2.0241754055023193, + "logits/rejected": -1.9385268688201904, + "logps/chosen": -263.27264404296875, + "logps/rejected": -232.6566162109375, + "loss": 0.5237, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.030437633395195007, + "rewards/margins": 0.69951331615448, + "rewards/rejected": -0.7299508452415466, + "step": 1880 + }, + { + "epoch": 1.95, + "learning_rate": 1.9402985074626865e-07, + "logits/chosen": -2.011044979095459, + "logits/rejected": -1.931051254272461, + "logps/chosen": -275.4610290527344, + "logps/rejected": -232.85659790039062, + "loss": 0.5614, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1333492547273636, + "rewards/margins": 0.5849149227142334, + "rewards/rejected": -0.7182641625404358, + "step": 1890 + }, + { + "epoch": 1.96, + "learning_rate": 1.921163413700727e-07, + "logits/chosen": -1.9748862981796265, + "logits/rejected": -1.8773696422576904, + "logps/chosen": -263.586669921875, + "logps/rejected": -236.25424194335938, + "loss": 0.5823, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.19906353950500488, + "rewards/margins": 0.5175008773803711, + "rewards/rejected": -0.7165643572807312, + "step": 1900 + }, + { + "epoch": 1.97, + "learning_rate": 1.9020283199387677e-07, + "logits/chosen": -2.0126466751098633, + "logits/rejected": -1.8699384927749634, + "logps/chosen": -262.9295959472656, + "logps/rejected": -229.73031616210938, + "loss": 0.5594, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.05735217407345772, + "rewards/margins": 0.637950599193573, + "rewards/rejected": -0.6953027248382568, + "step": 1910 + }, + { + "epoch": 1.98, + "learning_rate": 1.8828932261768083e-07, + "logits/chosen": -2.0402626991271973, + "logits/rejected": -1.917449712753296, + "logps/chosen": -268.4327392578125, + "logps/rejected": -221.59817504882812, + "loss": 0.5291, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.070868119597435, + "rewards/margins": 0.6999655365943909, + "rewards/rejected": -0.7708336114883423, + "step": 1920 + }, + { + "epoch": 1.99, + "learning_rate": 1.8637581324148487e-07, + "logits/chosen": -2.0629758834838867, + "logits/rejected": -1.9960095882415771, + "logps/chosen": -284.08978271484375, + "logps/rejected": -242.78659057617188, + "loss": 0.5491, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.10976877063512802, + "rewards/margins": 0.6353492140769958, + "rewards/rejected": -0.7451180219650269, + "step": 1930 + }, + { + "epoch": 2.0, + "eval_logits/chosen": -1.9162061214447021, + "eval_logits/rejected": -1.827931523323059, + "eval_logps/chosen": -283.52166748046875, + "eval_logps/rejected": -236.8552703857422, + "eval_loss": 0.5437692403793335, + "eval_rewards/accuracies": 0.7192460298538208, + "eval_rewards/chosen": -0.08363496512174606, + "eval_rewards/margins": 0.6414214968681335, + "eval_rewards/rejected": -0.7250563502311707, + "eval_runtime": 875.182, + "eval_samples_per_second": 2.285, + "eval_steps_per_second": 0.072, + "step": 1936 + }, + { + "epoch": 2.0, + "learning_rate": 1.8446230386528893e-07, + "logits/chosen": -2.0089569091796875, + "logits/rejected": -1.9302289485931396, + "logps/chosen": -247.7608184814453, + "logps/rejected": -208.72274780273438, + "loss": 0.5511, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.14254480600357056, + "rewards/margins": 0.5513399839401245, + "rewards/rejected": -0.6938849091529846, + "step": 1940 + }, + { + "epoch": 2.01, + "learning_rate": 1.82548794489093e-07, + "logits/chosen": -2.0011610984802246, + "logits/rejected": -1.95159113407135, + "logps/chosen": -274.76385498046875, + "logps/rejected": -252.4306182861328, + "loss": 0.5413, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.13799461722373962, + "rewards/margins": 0.6053641438484192, + "rewards/rejected": -0.7433587908744812, + "step": 1950 + }, + { + "epoch": 2.02, + "learning_rate": 1.8063528511289706e-07, + "logits/chosen": -1.971734642982483, + "logits/rejected": -1.9144260883331299, + "logps/chosen": -266.9194641113281, + "logps/rejected": -260.69573974609375, + "loss": 0.5458, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.11405863612890244, + "rewards/margins": 0.6057092547416687, + "rewards/rejected": -0.7197679281234741, + "step": 1960 + }, + { + "epoch": 2.03, + "learning_rate": 1.7872177573670112e-07, + "logits/chosen": -1.9765701293945312, + "logits/rejected": -1.987164855003357, + "logps/chosen": -261.4850158691406, + "logps/rejected": -214.2167510986328, + "loss": 0.5407, + "rewards/accuracies": 0.746874988079071, + "rewards/chosen": -0.06964612007141113, + "rewards/margins": 0.6810376644134521, + "rewards/rejected": -0.7506837844848633, + "step": 1970 + }, + { + "epoch": 2.04, + "learning_rate": 1.7680826636050515e-07, + "logits/chosen": -1.9535974264144897, + "logits/rejected": -1.8755991458892822, + "logps/chosen": -275.89776611328125, + "logps/rejected": -231.1304931640625, + "loss": 0.5352, + "rewards/accuracies": 0.7406250238418579, + "rewards/chosen": -0.09391085803508759, + "rewards/margins": 0.6898946762084961, + "rewards/rejected": -0.7838054895401001, + "step": 1980 + }, + { + "epoch": 2.06, + "learning_rate": 1.7489475698430921e-07, + "logits/chosen": -1.9550580978393555, + "logits/rejected": -1.9118692874908447, + "logps/chosen": -261.0256652832031, + "logps/rejected": -222.1853485107422, + "loss": 0.5543, + "rewards/accuracies": 0.734375, + "rewards/chosen": -0.10755525529384613, + "rewards/margins": 0.5950790643692017, + "rewards/rejected": -0.7026342749595642, + "step": 1990 + }, + { + "epoch": 2.07, + "learning_rate": 1.7298124760811328e-07, + "logits/chosen": -2.0680158138275146, + "logits/rejected": -1.8314135074615479, + "logps/chosen": -273.5735778808594, + "logps/rejected": -221.8857421875, + "loss": 0.5556, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.08592827618122101, + "rewards/margins": 0.7069720029830933, + "rewards/rejected": -0.7929002046585083, + "step": 2000 + }, + { + "epoch": 2.08, + "learning_rate": 1.7106773823191734e-07, + "logits/chosen": -2.041013240814209, + "logits/rejected": -1.9429569244384766, + "logps/chosen": -261.6728820800781, + "logps/rejected": -217.9615020751953, + "loss": 0.5335, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.09488911926746368, + "rewards/margins": 0.7252017259597778, + "rewards/rejected": -0.8200907707214355, + "step": 2010 + }, + { + "epoch": 2.09, + "learning_rate": 1.691542288557214e-07, + "logits/chosen": -1.9867607355117798, + "logits/rejected": -1.8171255588531494, + "logps/chosen": -277.5356750488281, + "logps/rejected": -232.9903106689453, + "loss": 0.5532, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12692232429981232, + "rewards/margins": 0.6040672063827515, + "rewards/rejected": -0.7309895753860474, + "step": 2020 + }, + { + "epoch": 2.1, + "learning_rate": 1.6724071947952544e-07, + "logits/chosen": -2.0031819343566895, + "logits/rejected": -1.9382264614105225, + "logps/chosen": -273.3340759277344, + "logps/rejected": -218.45809936523438, + "loss": 0.5437, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.13407178223133087, + "rewards/margins": 0.6485711932182312, + "rewards/rejected": -0.7826430201530457, + "step": 2030 + }, + { + "epoch": 2.11, + "learning_rate": 1.653272101033295e-07, + "logits/chosen": -1.94761061668396, + "logits/rejected": -1.8981701135635376, + "logps/chosen": -279.50396728515625, + "logps/rejected": -234.0538330078125, + "loss": 0.5158, + "rewards/accuracies": 0.753125011920929, + "rewards/chosen": -0.09492893517017365, + "rewards/margins": 0.7014483213424683, + "rewards/rejected": -0.7963773608207703, + "step": 2040 + }, + { + "epoch": 2.12, + "learning_rate": 1.6341370072713356e-07, + "logits/chosen": -2.0171449184417725, + "logits/rejected": -1.953629493713379, + "logps/chosen": -290.34478759765625, + "logps/rejected": -250.620849609375, + "loss": 0.5349, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.11246202141046524, + "rewards/margins": 0.6265016794204712, + "rewards/rejected": -0.738963782787323, + "step": 2050 + }, + { + "epoch": 2.13, + "learning_rate": 1.6150019135093762e-07, + "logits/chosen": -2.0697007179260254, + "logits/rejected": -1.931609869003296, + "logps/chosen": -298.30694580078125, + "logps/rejected": -233.61599731445312, + "loss": 0.539, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.08447974920272827, + "rewards/margins": 0.6712833046913147, + "rewards/rejected": -0.755763053894043, + "step": 2060 + }, + { + "epoch": 2.14, + "learning_rate": 1.5958668197474169e-07, + "logits/chosen": -2.0353074073791504, + "logits/rejected": -1.9606077671051025, + "logps/chosen": -266.8646545410156, + "logps/rejected": -252.83901977539062, + "loss": 0.5429, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.04690312594175339, + "rewards/margins": 0.5956650376319885, + "rewards/rejected": -0.6425682306289673, + "step": 2070 + }, + { + "epoch": 2.15, + "learning_rate": 1.5767317259854572e-07, + "logits/chosen": -1.9791412353515625, + "logits/rejected": -1.9253734350204468, + "logps/chosen": -278.138916015625, + "logps/rejected": -210.71810913085938, + "loss": 0.5448, + "rewards/accuracies": 0.7406250238418579, + "rewards/chosen": -0.008852415718138218, + "rewards/margins": 0.6631059646606445, + "rewards/rejected": -0.671958327293396, + "step": 2080 + }, + { + "epoch": 2.16, + "learning_rate": 1.5575966322234978e-07, + "logits/chosen": -1.9551115036010742, + "logits/rejected": -1.9077117443084717, + "logps/chosen": -258.12127685546875, + "logps/rejected": -231.0892333984375, + "loss": 0.5313, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.08477520197629929, + "rewards/margins": 0.6596347093582153, + "rewards/rejected": -0.74440997838974, + "step": 2090 + }, + { + "epoch": 2.17, + "learning_rate": 1.5384615384615385e-07, + "logits/chosen": -2.013572931289673, + "logits/rejected": -1.9160667657852173, + "logps/chosen": -266.0271911621094, + "logps/rejected": -214.8003387451172, + "loss": 0.5301, + "rewards/accuracies": 0.778124988079071, + "rewards/chosen": -0.12546849250793457, + "rewards/margins": 0.7021899223327637, + "rewards/rejected": -0.827658474445343, + "step": 2100 + }, + { + "epoch": 2.18, + "learning_rate": 1.519326444699579e-07, + "logits/chosen": -1.9861072301864624, + "logits/rejected": -1.8711364269256592, + "logps/chosen": -256.51129150390625, + "logps/rejected": -210.8562774658203, + "loss": 0.5306, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.11212404072284698, + "rewards/margins": 0.6418614387512207, + "rewards/rejected": -0.7539855241775513, + "step": 2110 + }, + { + "epoch": 2.19, + "learning_rate": 1.5001913509376197e-07, + "logits/chosen": -2.0319206714630127, + "logits/rejected": -1.9812395572662354, + "logps/chosen": -287.5851135253906, + "logps/rejected": -243.9886932373047, + "loss": 0.5534, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.15451140701770782, + "rewards/margins": 0.6129166483879089, + "rewards/rejected": -0.7674281001091003, + "step": 2120 + }, + { + "epoch": 2.2, + "learning_rate": 1.4810562571756603e-07, + "logits/chosen": -2.04410719871521, + "logits/rejected": -1.8800079822540283, + "logps/chosen": -247.6931610107422, + "logps/rejected": -227.32296752929688, + "loss": 0.5552, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.08454032987356186, + "rewards/margins": 0.7250841856002808, + "rewards/rejected": -0.8096244931221008, + "step": 2130 + }, + { + "epoch": 2.21, + "learning_rate": 1.4619211634137007e-07, + "logits/chosen": -1.9265568256378174, + "logits/rejected": -1.8220539093017578, + "logps/chosen": -256.62548828125, + "logps/rejected": -212.54470825195312, + "loss": 0.5332, + "rewards/accuracies": 0.753125011920929, + "rewards/chosen": -0.101638063788414, + "rewards/margins": 0.6411160230636597, + "rewards/rejected": -0.7427541613578796, + "step": 2140 + }, + { + "epoch": 2.22, + "learning_rate": 1.4427860696517413e-07, + "logits/chosen": -2.0046191215515137, + "logits/rejected": -1.8799793720245361, + "logps/chosen": -281.8408508300781, + "logps/rejected": -235.22567749023438, + "loss": 0.5653, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.11603154242038727, + "rewards/margins": 0.5813745260238647, + "rewards/rejected": -0.6974060535430908, + "step": 2150 + }, + { + "epoch": 2.23, + "learning_rate": 1.423650975889782e-07, + "logits/chosen": -2.0367672443389893, + "logits/rejected": -1.9846054315567017, + "logps/chosen": -287.5448303222656, + "logps/rejected": -236.52615356445312, + "loss": 0.5336, + "rewards/accuracies": 0.746874988079071, + "rewards/chosen": -0.05621805042028427, + "rewards/margins": 0.7016788721084595, + "rewards/rejected": -0.757896900177002, + "step": 2160 + }, + { + "epoch": 2.24, + "learning_rate": 1.4045158821278225e-07, + "logits/chosen": -1.9967548847198486, + "logits/rejected": -1.8803222179412842, + "logps/chosen": -260.0086975097656, + "logps/rejected": -229.26632690429688, + "loss": 0.5327, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.12442709505558014, + "rewards/margins": 0.7182163000106812, + "rewards/rejected": -0.8426433801651001, + "step": 2170 + }, + { + "epoch": 2.25, + "learning_rate": 1.3853807883658632e-07, + "logits/chosen": -2.06010103225708, + "logits/rejected": -1.979443907737732, + "logps/chosen": -284.96319580078125, + "logps/rejected": -228.4374237060547, + "loss": 0.5603, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.12428168207406998, + "rewards/margins": 0.5993887186050415, + "rewards/rejected": -0.7236703634262085, + "step": 2180 + }, + { + "epoch": 2.26, + "learning_rate": 1.3662456946039035e-07, + "logits/chosen": -1.9753601551055908, + "logits/rejected": -1.9079968929290771, + "logps/chosen": -248.7126922607422, + "logps/rejected": -220.49960327148438, + "loss": 0.5399, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.11765311658382416, + "rewards/margins": 0.5911535024642944, + "rewards/rejected": -0.7088066339492798, + "step": 2190 + }, + { + "epoch": 2.27, + "learning_rate": 1.3471106008419441e-07, + "logits/chosen": -1.9624128341674805, + "logits/rejected": -1.902761697769165, + "logps/chosen": -259.12322998046875, + "logps/rejected": -221.6943359375, + "loss": 0.5428, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.16727255284786224, + "rewards/margins": 0.5986199378967285, + "rewards/rejected": -0.7658926248550415, + "step": 2200 + }, + { + "epoch": 2.28, + "learning_rate": 1.3279755070799848e-07, + "logits/chosen": -2.0055718421936035, + "logits/rejected": -1.8739973306655884, + "logps/chosen": -271.47802734375, + "logps/rejected": -227.26358032226562, + "loss": 0.5605, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.1572277843952179, + "rewards/margins": 0.5724858045578003, + "rewards/rejected": -0.7297135591506958, + "step": 2210 + }, + { + "epoch": 2.29, + "learning_rate": 1.3088404133180254e-07, + "logits/chosen": -2.0300581455230713, + "logits/rejected": -1.9286314249038696, + "logps/chosen": -301.9215393066406, + "logps/rejected": -230.28164672851562, + "loss": 0.4922, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.0035975924693048, + "rewards/margins": 0.7717592716217041, + "rewards/rejected": -0.7753568291664124, + "step": 2220 + }, + { + "epoch": 2.3, + "learning_rate": 1.289705319556066e-07, + "logits/chosen": -1.9785239696502686, + "logits/rejected": -1.8718605041503906, + "logps/chosen": -260.2724609375, + "logps/rejected": -227.3611297607422, + "loss": 0.5343, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.11053315550088882, + "rewards/margins": 0.6837419271469116, + "rewards/rejected": -0.7942751049995422, + "step": 2230 + }, + { + "epoch": 2.31, + "learning_rate": 1.2705702257941064e-07, + "logits/chosen": -2.042748212814331, + "logits/rejected": -1.9830291271209717, + "logps/chosen": -286.2877502441406, + "logps/rejected": -233.0985870361328, + "loss": 0.5189, + "rewards/accuracies": 0.753125011920929, + "rewards/chosen": -0.12127647548913956, + "rewards/margins": 0.6687451004981995, + "rewards/rejected": -0.790021538734436, + "step": 2240 + }, + { + "epoch": 2.32, + "learning_rate": 1.251435132032147e-07, + "logits/chosen": -2.056126117706299, + "logits/rejected": -1.927869439125061, + "logps/chosen": -270.32427978515625, + "logps/rejected": -233.14218139648438, + "loss": 0.5377, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.08137933909893036, + "rewards/margins": 0.6417652368545532, + "rewards/rejected": -0.7231445908546448, + "step": 2250 + }, + { + "epoch": 2.33, + "learning_rate": 1.2323000382701873e-07, + "logits/chosen": -2.028714656829834, + "logits/rejected": -1.9313879013061523, + "logps/chosen": -268.63214111328125, + "logps/rejected": -234.97305297851562, + "loss": 0.5715, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.17626860737800598, + "rewards/margins": 0.5661368370056152, + "rewards/rejected": -0.7424054145812988, + "step": 2260 + }, + { + "epoch": 2.34, + "learning_rate": 1.213164944508228e-07, + "logits/chosen": -2.0209312438964844, + "logits/rejected": -1.9482758045196533, + "logps/chosen": -280.43505859375, + "logps/rejected": -251.8485107421875, + "loss": 0.5177, + "rewards/accuracies": 0.746874988079071, + "rewards/chosen": -0.16871343553066254, + "rewards/margins": 0.7220279574394226, + "rewards/rejected": -0.8907413482666016, + "step": 2270 + }, + { + "epoch": 2.35, + "learning_rate": 1.1940298507462686e-07, + "logits/chosen": -1.9852972030639648, + "logits/rejected": -1.9430131912231445, + "logps/chosen": -278.09869384765625, + "logps/rejected": -217.71533203125, + "loss": 0.5268, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.1097816452383995, + "rewards/margins": 0.7086385488510132, + "rewards/rejected": -0.8184202313423157, + "step": 2280 + }, + { + "epoch": 2.37, + "learning_rate": 1.1748947569843092e-07, + "logits/chosen": -2.027526378631592, + "logits/rejected": -1.938901662826538, + "logps/chosen": -276.50341796875, + "logps/rejected": -230.8983154296875, + "loss": 0.5165, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.07388017326593399, + "rewards/margins": 0.7378325462341309, + "rewards/rejected": -0.8117126226425171, + "step": 2290 + }, + { + "epoch": 2.38, + "learning_rate": 1.1557596632223497e-07, + "logits/chosen": -2.0729262828826904, + "logits/rejected": -1.900803565979004, + "logps/chosen": -271.7540283203125, + "logps/rejected": -227.78384399414062, + "loss": 0.5282, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.07709081470966339, + "rewards/margins": 0.7070693969726562, + "rewards/rejected": -0.7841601967811584, + "step": 2300 + }, + { + "epoch": 2.39, + "learning_rate": 1.1366245694603903e-07, + "logits/chosen": -1.99917471408844, + "logits/rejected": -1.9048084020614624, + "logps/chosen": -258.48968505859375, + "logps/rejected": -220.8121337890625, + "loss": 0.5524, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.14988255500793457, + "rewards/margins": 0.5792022943496704, + "rewards/rejected": -0.729084849357605, + "step": 2310 + }, + { + "epoch": 2.4, + "learning_rate": 1.1174894756984308e-07, + "logits/chosen": -1.9502952098846436, + "logits/rejected": -1.9144408702850342, + "logps/chosen": -243.72299194335938, + "logps/rejected": -227.6197967529297, + "loss": 0.528, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.16638337075710297, + "rewards/margins": 0.5764808654785156, + "rewards/rejected": -0.7428642511367798, + "step": 2320 + }, + { + "epoch": 2.41, + "learning_rate": 1.0983543819364714e-07, + "logits/chosen": -2.0321359634399414, + "logits/rejected": -1.9428303241729736, + "logps/chosen": -265.5797119140625, + "logps/rejected": -238.43368530273438, + "loss": 0.4978, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.11228694766759872, + "rewards/margins": 0.6872363090515137, + "rewards/rejected": -0.7995232343673706, + "step": 2330 + }, + { + "epoch": 2.42, + "learning_rate": 1.079219288174512e-07, + "logits/chosen": -1.9360284805297852, + "logits/rejected": -1.8506568670272827, + "logps/chosen": -245.03762817382812, + "logps/rejected": -236.2658233642578, + "loss": 0.5417, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.12498140335083008, + "rewards/margins": 0.6189488172531128, + "rewards/rejected": -0.7439301609992981, + "step": 2340 + }, + { + "epoch": 2.43, + "learning_rate": 1.0600841944125525e-07, + "logits/chosen": -1.9778602123260498, + "logits/rejected": -1.8423837423324585, + "logps/chosen": -260.97308349609375, + "logps/rejected": -221.7749786376953, + "loss": 0.5554, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.19020505249500275, + "rewards/margins": 0.5690335631370544, + "rewards/rejected": -0.759238600730896, + "step": 2350 + }, + { + "epoch": 2.44, + "learning_rate": 1.0409491006505931e-07, + "logits/chosen": -2.021061897277832, + "logits/rejected": -1.9514410495758057, + "logps/chosen": -264.15435791015625, + "logps/rejected": -229.5314483642578, + "loss": 0.5351, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.0817098617553711, + "rewards/margins": 0.7072576880455017, + "rewards/rejected": -0.7889676094055176, + "step": 2360 + }, + { + "epoch": 2.45, + "learning_rate": 1.0218140068886336e-07, + "logits/chosen": -1.971008062362671, + "logits/rejected": -1.8351669311523438, + "logps/chosen": -272.0447082519531, + "logps/rejected": -224.9204559326172, + "loss": 0.5529, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.10915365070104599, + "rewards/margins": 0.6357630491256714, + "rewards/rejected": -0.7449167370796204, + "step": 2370 + }, + { + "epoch": 2.46, + "learning_rate": 1.0026789131266743e-07, + "logits/chosen": -1.9370200634002686, + "logits/rejected": -1.8351131677627563, + "logps/chosen": -282.9969177246094, + "logps/rejected": -241.0260772705078, + "loss": 0.5493, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.0709105059504509, + "rewards/margins": 0.6791272163391113, + "rewards/rejected": -0.7500376105308533, + "step": 2380 + }, + { + "epoch": 2.47, + "learning_rate": 9.835438193647149e-08, + "logits/chosen": -1.9906991720199585, + "logits/rejected": -1.9335302114486694, + "logps/chosen": -265.366455078125, + "logps/rejected": -219.4970245361328, + "loss": 0.5563, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.1341801881790161, + "rewards/margins": 0.6088107824325562, + "rewards/rejected": -0.7429909706115723, + "step": 2390 + }, + { + "epoch": 2.48, + "learning_rate": 9.644087256027554e-08, + "logits/chosen": -1.961403250694275, + "logits/rejected": -1.9385201930999756, + "logps/chosen": -272.18280029296875, + "logps/rejected": -234.25277709960938, + "loss": 0.5334, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.10721113532781601, + "rewards/margins": 0.6185725927352905, + "rewards/rejected": -0.7257837057113647, + "step": 2400 + }, + { + "epoch": 2.49, + "learning_rate": 9.45273631840796e-08, + "logits/chosen": -2.0202138423919678, + "logits/rejected": -1.9500621557235718, + "logps/chosen": -258.01361083984375, + "logps/rejected": -228.53732299804688, + "loss": 0.5474, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.13280650973320007, + "rewards/margins": 0.6008110046386719, + "rewards/rejected": -0.7336176037788391, + "step": 2410 + }, + { + "epoch": 2.5, + "learning_rate": 9.261385380788366e-08, + "logits/chosen": -1.9771407842636108, + "logits/rejected": -1.956167459487915, + "logps/chosen": -261.42169189453125, + "logps/rejected": -235.85366821289062, + "loss": 0.5367, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.12852981686592102, + "rewards/margins": 0.5638247132301331, + "rewards/rejected": -0.6923545598983765, + "step": 2420 + }, + { + "epoch": 2.51, + "learning_rate": 9.070034443168771e-08, + "logits/chosen": -2.0048177242279053, + "logits/rejected": -1.9468837976455688, + "logps/chosen": -258.286376953125, + "logps/rejected": -239.4003448486328, + "loss": 0.557, + "rewards/accuracies": 0.7406250238418579, + "rewards/chosen": -0.10692036151885986, + "rewards/margins": 0.6991198658943176, + "rewards/rejected": -0.8060401678085327, + "step": 2430 + }, + { + "epoch": 2.52, + "learning_rate": 8.878683505549177e-08, + "logits/chosen": -2.012662410736084, + "logits/rejected": -1.9248571395874023, + "logps/chosen": -260.6874694824219, + "logps/rejected": -243.2574005126953, + "loss": 0.5111, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.09256302565336227, + "rewards/margins": 0.6950146555900574, + "rewards/rejected": -0.7875776886940002, + "step": 2440 + }, + { + "epoch": 2.53, + "learning_rate": 8.687332567929582e-08, + "logits/chosen": -1.9357149600982666, + "logits/rejected": -1.927011489868164, + "logps/chosen": -264.5352478027344, + "logps/rejected": -239.3921356201172, + "loss": 0.5381, + "rewards/accuracies": 0.753125011920929, + "rewards/chosen": -0.1247233971953392, + "rewards/margins": 0.6627745032310486, + "rewards/rejected": -0.7874979972839355, + "step": 2450 + }, + { + "epoch": 2.54, + "learning_rate": 8.495981630309988e-08, + "logits/chosen": -1.9526309967041016, + "logits/rejected": -1.9026234149932861, + "logps/chosen": -281.9859924316406, + "logps/rejected": -228.44052124023438, + "loss": 0.5448, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13007353246212006, + "rewards/margins": 0.6389485001564026, + "rewards/rejected": -0.7690221071243286, + "step": 2460 + }, + { + "epoch": 2.55, + "learning_rate": 8.304630692690395e-08, + "logits/chosen": -2.0521068572998047, + "logits/rejected": -1.9333775043487549, + "logps/chosen": -267.8842468261719, + "logps/rejected": -223.2080535888672, + "loss": 0.5419, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.09863010793924332, + "rewards/margins": 0.6975767016410828, + "rewards/rejected": -0.7962067723274231, + "step": 2470 + }, + { + "epoch": 2.56, + "learning_rate": 8.1132797550708e-08, + "logits/chosen": -2.065293073654175, + "logits/rejected": -1.9522087574005127, + "logps/chosen": -302.3513488769531, + "logps/rejected": -239.3452606201172, + "loss": 0.5312, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.041663073003292084, + "rewards/margins": 0.7064257264137268, + "rewards/rejected": -0.7480887770652771, + "step": 2480 + }, + { + "epoch": 2.57, + "learning_rate": 7.921928817451206e-08, + "logits/chosen": -2.0390076637268066, + "logits/rejected": -1.9214242696762085, + "logps/chosen": -267.5995788574219, + "logps/rejected": -236.94271850585938, + "loss": 0.5194, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.10287908464670181, + "rewards/margins": 0.7216282486915588, + "rewards/rejected": -0.8245073556900024, + "step": 2490 + }, + { + "epoch": 2.58, + "learning_rate": 7.73057787983161e-08, + "logits/chosen": -1.987099051475525, + "logits/rejected": -1.9656906127929688, + "logps/chosen": -286.71746826171875, + "logps/rejected": -242.8203887939453, + "loss": 0.5309, + "rewards/accuracies": 0.765625, + "rewards/chosen": -0.08397471904754639, + "rewards/margins": 0.6902012825012207, + "rewards/rejected": -0.7741760015487671, + "step": 2500 + }, + { + "epoch": 2.59, + "learning_rate": 7.539226942212017e-08, + "logits/chosen": -1.9945751428604126, + "logits/rejected": -1.8857038021087646, + "logps/chosen": -256.72479248046875, + "logps/rejected": -226.22207641601562, + "loss": 0.5227, + "rewards/accuracies": 0.753125011920929, + "rewards/chosen": -0.1443193554878235, + "rewards/margins": 0.7128327488899231, + "rewards/rejected": -0.8571521639823914, + "step": 2510 + }, + { + "epoch": 2.6, + "learning_rate": 7.347876004592423e-08, + "logits/chosen": -2.042407512664795, + "logits/rejected": -1.8979930877685547, + "logps/chosen": -257.240478515625, + "logps/rejected": -211.95138549804688, + "loss": 0.5415, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.10604407638311386, + "rewards/margins": 0.6722184419631958, + "rewards/rejected": -0.7782624363899231, + "step": 2520 + }, + { + "epoch": 2.61, + "learning_rate": 7.156525066972828e-08, + "logits/chosen": -1.9255244731903076, + "logits/rejected": -1.905170202255249, + "logps/chosen": -253.9196319580078, + "logps/rejected": -221.67984008789062, + "loss": 0.5594, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.1309424787759781, + "rewards/margins": 0.5638185739517212, + "rewards/rejected": -0.6947609782218933, + "step": 2530 + }, + { + "epoch": 2.62, + "learning_rate": 6.965174129353234e-08, + "logits/chosen": -1.9832531213760376, + "logits/rejected": -1.9322866201400757, + "logps/chosen": -277.56982421875, + "logps/rejected": -221.8081512451172, + "loss": 0.5328, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.05555805563926697, + "rewards/margins": 0.7537458539009094, + "rewards/rejected": -0.8093039393424988, + "step": 2540 + }, + { + "epoch": 2.63, + "learning_rate": 6.773823191733639e-08, + "logits/chosen": -1.9376776218414307, + "logits/rejected": -1.8852014541625977, + "logps/chosen": -259.39947509765625, + "logps/rejected": -222.1279296875, + "loss": 0.5364, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.07750894874334335, + "rewards/margins": 0.7031437754631042, + "rewards/rejected": -0.780652642250061, + "step": 2550 + }, + { + "epoch": 2.64, + "learning_rate": 6.582472254114045e-08, + "logits/chosen": -1.945090889930725, + "logits/rejected": -1.9398300647735596, + "logps/chosen": -260.34881591796875, + "logps/rejected": -241.45230102539062, + "loss": 0.5712, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.19983193278312683, + "rewards/margins": 0.5203108787536621, + "rewards/rejected": -0.7201427221298218, + "step": 2560 + }, + { + "epoch": 2.65, + "learning_rate": 6.391121316494451e-08, + "logits/chosen": -1.9702688455581665, + "logits/rejected": -1.9331938028335571, + "logps/chosen": -273.8888854980469, + "logps/rejected": -226.31692504882812, + "loss": 0.5264, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.11131913959980011, + "rewards/margins": 0.6655842065811157, + "rewards/rejected": -0.7769032716751099, + "step": 2570 + }, + { + "epoch": 2.66, + "learning_rate": 6.199770378874856e-08, + "logits/chosen": -1.9572299718856812, + "logits/rejected": -1.9173994064331055, + "logps/chosen": -266.03692626953125, + "logps/rejected": -233.6370849609375, + "loss": 0.5311, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.05556722730398178, + "rewards/margins": 0.6352362036705017, + "rewards/rejected": -0.6908034086227417, + "step": 2580 + }, + { + "epoch": 2.67, + "learning_rate": 6.008419441255262e-08, + "logits/chosen": -1.973695993423462, + "logits/rejected": -1.9042612314224243, + "logps/chosen": -282.314208984375, + "logps/rejected": -224.1729736328125, + "loss": 0.5393, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.08098714798688889, + "rewards/margins": 0.6813811659812927, + "rewards/rejected": -0.762368381023407, + "step": 2590 + }, + { + "epoch": 2.69, + "learning_rate": 5.817068503635668e-08, + "logits/chosen": -2.041367769241333, + "logits/rejected": -1.9625844955444336, + "logps/chosen": -278.90740966796875, + "logps/rejected": -237.153076171875, + "loss": 0.5267, + "rewards/accuracies": 0.734375, + "rewards/chosen": -0.08425331115722656, + "rewards/margins": 0.6965413093566895, + "rewards/rejected": -0.780794620513916, + "step": 2600 + }, + { + "epoch": 2.7, + "learning_rate": 5.6257175660160735e-08, + "logits/chosen": -1.9530586004257202, + "logits/rejected": -1.9685173034667969, + "logps/chosen": -242.7851104736328, + "logps/rejected": -225.36294555664062, + "loss": 0.521, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.10606219619512558, + "rewards/margins": 0.6712923645973206, + "rewards/rejected": -0.7773545384407043, + "step": 2610 + }, + { + "epoch": 2.71, + "learning_rate": 5.4343666283964784e-08, + "logits/chosen": -1.9664418697357178, + "logits/rejected": -1.8895866870880127, + "logps/chosen": -274.50543212890625, + "logps/rejected": -228.39794921875, + "loss": 0.5143, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12829892337322235, + "rewards/margins": 0.7127763032913208, + "rewards/rejected": -0.8410751223564148, + "step": 2620 + }, + { + "epoch": 2.72, + "learning_rate": 5.243015690776884e-08, + "logits/chosen": -1.9619324207305908, + "logits/rejected": -1.8757219314575195, + "logps/chosen": -283.93951416015625, + "logps/rejected": -227.21240234375, + "loss": 0.525, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.09419764578342438, + "rewards/margins": 0.6821730136871338, + "rewards/rejected": -0.776370644569397, + "step": 2630 + }, + { + "epoch": 2.73, + "learning_rate": 5.05166475315729e-08, + "logits/chosen": -1.965478539466858, + "logits/rejected": -1.9349143505096436, + "logps/chosen": -277.0966796875, + "logps/rejected": -230.806640625, + "loss": 0.5439, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.1608276665210724, + "rewards/margins": 0.6512233018875122, + "rewards/rejected": -0.812050998210907, + "step": 2640 + }, + { + "epoch": 2.74, + "learning_rate": 4.860313815537696e-08, + "logits/chosen": -1.9939587116241455, + "logits/rejected": -1.9180082082748413, + "logps/chosen": -264.33380126953125, + "logps/rejected": -241.66598510742188, + "loss": 0.5395, + "rewards/accuracies": 0.784375011920929, + "rewards/chosen": -0.09295650571584702, + "rewards/margins": 0.7515963315963745, + "rewards/rejected": -0.8445528745651245, + "step": 2650 + }, + { + "epoch": 2.75, + "learning_rate": 4.668962877918101e-08, + "logits/chosen": -1.9551360607147217, + "logits/rejected": -1.897792100906372, + "logps/chosen": -246.9778594970703, + "logps/rejected": -241.43692016601562, + "loss": 0.5288, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.14015261828899384, + "rewards/margins": 0.6561464071273804, + "rewards/rejected": -0.7962990999221802, + "step": 2660 + }, + { + "epoch": 2.76, + "learning_rate": 4.477611940298507e-08, + "logits/chosen": -2.035179615020752, + "logits/rejected": -1.9448999166488647, + "logps/chosen": -280.6395568847656, + "logps/rejected": -226.279541015625, + "loss": 0.5099, + "rewards/accuracies": 0.778124988079071, + "rewards/chosen": -0.013490036129951477, + "rewards/margins": 0.8554097414016724, + "rewards/rejected": -0.8688998222351074, + "step": 2670 + }, + { + "epoch": 2.77, + "learning_rate": 4.2862610026789124e-08, + "logits/chosen": -2.105760097503662, + "logits/rejected": -1.9209930896759033, + "logps/chosen": -263.80010986328125, + "logps/rejected": -233.33126831054688, + "loss": 0.5576, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.13011452555656433, + "rewards/margins": 0.6017405390739441, + "rewards/rejected": -0.731855034828186, + "step": 2680 + }, + { + "epoch": 2.78, + "learning_rate": 4.0949100650593186e-08, + "logits/chosen": -1.9066162109375, + "logits/rejected": -1.8646472692489624, + "logps/chosen": -257.67529296875, + "logps/rejected": -236.4352569580078, + "loss": 0.5459, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.17842397093772888, + "rewards/margins": 0.5756587982177734, + "rewards/rejected": -0.7540827393531799, + "step": 2690 + }, + { + "epoch": 2.79, + "learning_rate": 3.903559127439724e-08, + "logits/chosen": -2.0057380199432373, + "logits/rejected": -1.9169094562530518, + "logps/chosen": -273.3626403808594, + "logps/rejected": -229.9825439453125, + "loss": 0.5079, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.052403002977371216, + "rewards/margins": 0.7288587689399719, + "rewards/rejected": -0.7812617421150208, + "step": 2700 + }, + { + "epoch": 2.8, + "learning_rate": 3.71220818982013e-08, + "logits/chosen": -2.037144899368286, + "logits/rejected": -1.9049354791641235, + "logps/chosen": -272.304443359375, + "logps/rejected": -238.1118621826172, + "loss": 0.5517, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.2206374704837799, + "rewards/margins": 0.542033314704895, + "rewards/rejected": -0.7626706957817078, + "step": 2710 + }, + { + "epoch": 2.81, + "learning_rate": 3.520857252200535e-08, + "logits/chosen": -1.9005661010742188, + "logits/rejected": -1.87981379032135, + "logps/chosen": -263.9569091796875, + "logps/rejected": -239.2077178955078, + "loss": 0.539, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.11894295364618301, + "rewards/margins": 0.5795784592628479, + "rewards/rejected": -0.6985214352607727, + "step": 2720 + }, + { + "epoch": 2.82, + "learning_rate": 3.3295063145809414e-08, + "logits/chosen": -1.9351590871810913, + "logits/rejected": -1.9006996154785156, + "logps/chosen": -270.80377197265625, + "logps/rejected": -246.0286102294922, + "loss": 0.5474, + "rewards/accuracies": 0.778124988079071, + "rewards/chosen": -0.13235792517662048, + "rewards/margins": 0.6689977645874023, + "rewards/rejected": -0.8013556599617004, + "step": 2730 + }, + { + "epoch": 2.83, + "learning_rate": 3.138155376961347e-08, + "logits/chosen": -2.017043113708496, + "logits/rejected": -1.929476022720337, + "logps/chosen": -285.06182861328125, + "logps/rejected": -244.6565399169922, + "loss": 0.5623, + "rewards/accuracies": 0.746874988079071, + "rewards/chosen": -0.10070965439081192, + "rewards/margins": 0.6912192702293396, + "rewards/rejected": -0.7919288873672485, + "step": 2740 + }, + { + "epoch": 2.84, + "learning_rate": 2.9468044393417525e-08, + "logits/chosen": -1.9757921695709229, + "logits/rejected": -1.8634798526763916, + "logps/chosen": -261.4624938964844, + "logps/rejected": -230.32046508789062, + "loss": 0.5245, + "rewards/accuracies": 0.765625, + "rewards/chosen": -0.07247430086135864, + "rewards/margins": 0.7779797315597534, + "rewards/rejected": -0.8504541516304016, + "step": 2750 + }, + { + "epoch": 2.85, + "learning_rate": 2.755453501722158e-08, + "logits/chosen": -1.9504101276397705, + "logits/rejected": -1.9742431640625, + "logps/chosen": -264.0960998535156, + "logps/rejected": -229.517822265625, + "loss": 0.5135, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.09173519909381866, + "rewards/margins": 0.7189399003982544, + "rewards/rejected": -0.8106750249862671, + "step": 2760 + }, + { + "epoch": 2.86, + "learning_rate": 2.564102564102564e-08, + "logits/chosen": -2.043729305267334, + "logits/rejected": -1.9805526733398438, + "logps/chosen": -265.21710205078125, + "logps/rejected": -245.47158813476562, + "loss": 0.5323, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.1146395206451416, + "rewards/margins": 0.673359751701355, + "rewards/rejected": -0.7879992723464966, + "step": 2770 + }, + { + "epoch": 2.87, + "learning_rate": 2.3727516264829695e-08, + "logits/chosen": -2.017853021621704, + "logits/rejected": -1.9051122665405273, + "logps/chosen": -283.5233459472656, + "logps/rejected": -237.43875122070312, + "loss": 0.5349, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.0835433155298233, + "rewards/margins": 0.6877850294113159, + "rewards/rejected": -0.7713284492492676, + "step": 2780 + }, + { + "epoch": 2.88, + "learning_rate": 2.1814006888633754e-08, + "logits/chosen": -1.9762601852416992, + "logits/rejected": -1.9173065423965454, + "logps/chosen": -271.5351257324219, + "logps/rejected": -227.7848663330078, + "loss": 0.5481, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.10751036554574966, + "rewards/margins": 0.6145915389060974, + "rewards/rejected": -0.7221018671989441, + "step": 2790 + }, + { + "epoch": 2.89, + "learning_rate": 1.990049751243781e-08, + "logits/chosen": -2.0152127742767334, + "logits/rejected": -1.894903540611267, + "logps/chosen": -277.72955322265625, + "logps/rejected": -226.2139434814453, + "loss": 0.5381, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.09227684885263443, + "rewards/margins": 0.6984345316886902, + "rewards/rejected": -0.7907114028930664, + "step": 2800 + }, + { + "epoch": 2.9, + "learning_rate": 1.7986988136241865e-08, + "logits/chosen": -1.8761422634124756, + "logits/rejected": -1.8549368381500244, + "logps/chosen": -274.33721923828125, + "logps/rejected": -242.8546600341797, + "loss": 0.5018, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.08238744735717773, + "rewards/margins": 0.722454309463501, + "rewards/rejected": -0.8048418164253235, + "step": 2810 + }, + { + "epoch": 2.91, + "learning_rate": 1.6073478760045924e-08, + "logits/chosen": -1.9757686853408813, + "logits/rejected": -1.8756663799285889, + "logps/chosen": -282.96954345703125, + "logps/rejected": -235.4677734375, + "loss": 0.5485, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.16583535075187683, + "rewards/margins": 0.7119665145874023, + "rewards/rejected": -0.877801775932312, + "step": 2820 + }, + { + "epoch": 2.92, + "learning_rate": 1.4159969383849981e-08, + "logits/chosen": -1.966698408126831, + "logits/rejected": -1.959288239479065, + "logps/chosen": -271.11260986328125, + "logps/rejected": -223.1236572265625, + "loss": 0.5281, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08993484824895859, + "rewards/margins": 0.6874836087226868, + "rewards/rejected": -0.7774184346199036, + "step": 2830 + }, + { + "epoch": 2.93, + "learning_rate": 1.2246460007654037e-08, + "logits/chosen": -1.9788850545883179, + "logits/rejected": -1.9086517095565796, + "logps/chosen": -266.74066162109375, + "logps/rejected": -227.94088745117188, + "loss": 0.5735, + "rewards/accuracies": 0.659375011920929, + "rewards/chosen": -0.18117864429950714, + "rewards/margins": 0.4861881136894226, + "rewards/rejected": -0.6673667430877686, + "step": 2840 + }, + { + "epoch": 2.94, + "learning_rate": 1.0332950631458094e-08, + "logits/chosen": -1.937133550643921, + "logits/rejected": -1.9378074407577515, + "logps/chosen": -265.6098937988281, + "logps/rejected": -227.0642547607422, + "loss": 0.5439, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.0965404137969017, + "rewards/margins": 0.6471257209777832, + "rewards/rejected": -0.7436660528182983, + "step": 2850 + }, + { + "epoch": 2.95, + "learning_rate": 8.419441255262151e-09, + "logits/chosen": -1.9160661697387695, + "logits/rejected": -1.9068663120269775, + "logps/chosen": -261.45025634765625, + "logps/rejected": -236.23446655273438, + "loss": 0.5177, + "rewards/accuracies": 0.753125011920929, + "rewards/chosen": -0.1480577141046524, + "rewards/margins": 0.7233497500419617, + "rewards/rejected": -0.8714075088500977, + "step": 2860 + }, + { + "epoch": 2.96, + "learning_rate": 6.505931879066207e-09, + "logits/chosen": -1.9042994976043701, + "logits/rejected": -1.9673118591308594, + "logps/chosen": -255.11904907226562, + "logps/rejected": -217.8560333251953, + "loss": 0.5419, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2056998312473297, + "rewards/margins": 0.5817626714706421, + "rewards/rejected": -0.7874624729156494, + "step": 2870 + }, + { + "epoch": 2.97, + "learning_rate": 4.592422502870264e-09, + "logits/chosen": -2.018409013748169, + "logits/rejected": -1.9491459131240845, + "logps/chosen": -264.1302185058594, + "logps/rejected": -239.7041015625, + "loss": 0.5506, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.13123063743114471, + "rewards/margins": 0.5559999346733093, + "rewards/rejected": -0.6872305870056152, + "step": 2880 + }, + { + "epoch": 2.98, + "learning_rate": 2.6789131266743202e-09, + "logits/chosen": -1.965376853942871, + "logits/rejected": -1.8747295141220093, + "logps/chosen": -270.21197509765625, + "logps/rejected": -235.61495971679688, + "loss": 0.5223, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.11224790662527084, + "rewards/margins": 0.7009861469268799, + "rewards/rejected": -0.8132340312004089, + "step": 2890 + }, + { + "epoch": 3.0, + "learning_rate": 7.654037504783773e-10, + "logits/chosen": -1.9493694305419922, + "logits/rejected": -1.9246546030044556, + "logps/chosen": -252.29348754882812, + "logps/rejected": -249.4759521484375, + "loss": 0.5463, + "rewards/accuracies": 0.7406250238418579, + "rewards/chosen": -0.1315115988254547, + "rewards/margins": 0.6227953433990479, + "rewards/rejected": -0.754306972026825, + "step": 2900 + }, + { + "epoch": 3.0, + "eval_logits/chosen": -1.9102067947387695, + "eval_logits/rejected": -1.8216071128845215, + "eval_logps/chosen": -283.6626281738281, + "eval_logps/rejected": -237.35032653808594, + "eval_loss": 0.5380748510360718, + "eval_rewards/accuracies": 0.7182539701461792, + "eval_rewards/chosen": -0.09773208945989609, + "eval_rewards/margins": 0.6768320798873901, + "eval_rewards/rejected": -0.7745641469955444, + "eval_runtime": 872.6678, + "eval_samples_per_second": 2.292, + "eval_steps_per_second": 0.072, + "step": 2904 + }, + { + "epoch": 3.0, + "step": 2904, + "total_flos": 0.0, + "train_loss": 0.5707186016333662, + "train_runtime": 142661.7128, + "train_samples_per_second": 1.303, + "train_steps_per_second": 0.02 + } + ], + "logging_steps": 10, + "max_steps": 2904, + "num_train_epochs": 3, + "save_steps": 500, + "total_flos": 0.0, + "trial_name": null, + "trial_params": null +}