diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,7111 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9981298423724285, + "eval_steps": 100, + "global_step": 467, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0021373230029388193, + "grad_norm": 5.562169562786842, + "learning_rate": 1.4893617021276595e-08, + "logits/chosen": -0.9639219045639038, + "logits/rejected": -0.9681637287139893, + "logps/chosen": -254.6912078857422, + "logps/rejected": -263.5589904785156, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.004274646005877639, + "grad_norm": 5.450588274868951, + "learning_rate": 2.978723404255319e-08, + "logits/chosen": -0.9656609296798706, + "logits/rejected": -0.9244194030761719, + "logps/chosen": -326.766845703125, + "logps/rejected": -278.7445068359375, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 2 + }, + { + "epoch": 0.006411969008816457, + "grad_norm": 5.947914793856331, + "learning_rate": 4.468085106382978e-08, + "logits/chosen": -0.8438385725021362, + "logits/rejected": -0.8096433877944946, + "logps/chosen": -235.20404052734375, + "logps/rejected": -236.10145568847656, + "loss": 0.6932, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.0013289405032992363, + "rewards/margins": -0.00010445114457979798, + "rewards/rejected": 0.0014333915896713734, + "step": 3 + }, + { + "epoch": 0.008549292011755277, + "grad_norm": 5.296514371602818, + "learning_rate": 5.957446808510638e-08, + "logits/chosen": -0.9574068784713745, + "logits/rejected": -0.9784606099128723, + "logps/chosen": -205.1729736328125, + "logps/rejected": -211.75196838378906, + "loss": 0.6935, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.0005042314296588302, + "rewards/margins": -0.00139321805909276, + "rewards/rejected": 0.0008889864548109472, + "step": 4 + }, + { + "epoch": 0.010686615014694095, + "grad_norm": 5.901392519236585, + "learning_rate": 7.446808510638298e-08, + "logits/chosen": -1.0202012062072754, + "logits/rejected": -1.05290687084198, + "logps/chosen": -223.4305419921875, + "logps/rejected": -223.0294647216797, + "loss": 0.6933, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0012667321134358644, + "rewards/margins": 0.00010157108772546053, + "rewards/rejected": 0.0011651611421257257, + "step": 5 + }, + { + "epoch": 0.012823938017632914, + "grad_norm": 6.106968331942851, + "learning_rate": 8.936170212765956e-08, + "logits/chosen": -1.01454496383667, + "logits/rejected": -0.9522863626480103, + "logps/chosen": -310.51141357421875, + "logps/rejected": -288.7611083984375, + "loss": 0.6928, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.0014815283939242363, + "rewards/margins": -0.0021853544749319553, + "rewards/rejected": 0.0007038259063847363, + "step": 6 + }, + { + "epoch": 0.014961261020571734, + "grad_norm": 4.989592008812262, + "learning_rate": 1.0425531914893615e-07, + "logits/chosen": -1.084510087966919, + "logits/rejected": -1.029686689376831, + "logps/chosen": -236.44857788085938, + "logps/rejected": -227.99609375, + "loss": 0.6936, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.0002356480690650642, + "rewards/margins": 0.001065850374288857, + "rewards/rejected": -0.0013014983851462603, + "step": 7 + }, + { + "epoch": 0.017098584023510555, + "grad_norm": 5.864652813608174, + "learning_rate": 1.1914893617021276e-07, + "logits/chosen": -1.0324242115020752, + "logits/rejected": -0.9804142713546753, + "logps/chosen": -248.21893310546875, + "logps/rejected": -262.17919921875, + "loss": 0.6934, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0010573435574769974, + "rewards/margins": 7.791988173266873e-05, + "rewards/rejected": -0.001135263592004776, + "step": 8 + }, + { + "epoch": 0.01923590702644937, + "grad_norm": 5.012836085935026, + "learning_rate": 1.3404255319148934e-07, + "logits/chosen": -0.9063066244125366, + "logits/rejected": -0.888552188873291, + "logps/chosen": -281.49481201171875, + "logps/rejected": -285.80462646484375, + "loss": 0.6932, + "rewards/accuracies": 0.4375, + "rewards/chosen": 0.0012888526543974876, + "rewards/margins": -0.00021493433450814337, + "rewards/rejected": 0.0015037870034575462, + "step": 9 + }, + { + "epoch": 0.02137323002938819, + "grad_norm": 5.441686883546068, + "learning_rate": 1.4893617021276595e-07, + "logits/chosen": -1.1913858652114868, + "logits/rejected": -1.1670382022857666, + "logps/chosen": -264.4725646972656, + "logps/rejected": -285.6479797363281, + "loss": 0.6932, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.0027979514561593533, + "rewards/margins": 0.0013645172584801912, + "rewards/rejected": 0.0014334343140944839, + "step": 10 + }, + { + "epoch": 0.02351055303232701, + "grad_norm": 5.606555817511537, + "learning_rate": 1.6382978723404256e-07, + "logits/chosen": -1.0321263074874878, + "logits/rejected": -1.0099542140960693, + "logps/chosen": -266.173828125, + "logps/rejected": -278.39117431640625, + "loss": 0.6932, + "rewards/accuracies": 0.5, + "rewards/chosen": -5.334848538041115e-05, + "rewards/margins": 0.0002821112284436822, + "rewards/rejected": -0.00033545983023941517, + "step": 11 + }, + { + "epoch": 0.02564787603526583, + "grad_norm": 4.68531837452694, + "learning_rate": 1.7872340425531912e-07, + "logits/chosen": -0.9241148233413696, + "logits/rejected": -0.9032956957817078, + "logps/chosen": -232.3054962158203, + "logps/rejected": -213.7734375, + "loss": 0.6926, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.0020639896392822266, + "rewards/margins": 0.00174471375066787, + "rewards/rejected": 0.000319275917718187, + "step": 12 + }, + { + "epoch": 0.027785199038204648, + "grad_norm": 6.413087644872293, + "learning_rate": 1.9361702127659575e-07, + "logits/chosen": -0.8520964980125427, + "logits/rejected": -0.7619133591651917, + "logps/chosen": -257.8199462890625, + "logps/rejected": -286.0062255859375, + "loss": 0.6932, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.0032129527535289526, + "rewards/margins": 0.004196567460894585, + "rewards/rejected": -0.007409519515931606, + "step": 13 + }, + { + "epoch": 0.029922522041143467, + "grad_norm": 6.109488078686547, + "learning_rate": 2.085106382978723e-07, + "logits/chosen": -1.0061469078063965, + "logits/rejected": -1.0426419973373413, + "logps/chosen": -300.9575500488281, + "logps/rejected": -284.1053466796875, + "loss": 0.6933, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.0033643674105405807, + "rewards/margins": -0.002198734087869525, + "rewards/rejected": -0.001165633206255734, + "step": 14 + }, + { + "epoch": 0.03205984504408229, + "grad_norm": 5.1906516925117225, + "learning_rate": 2.2340425531914894e-07, + "logits/chosen": -1.1642401218414307, + "logits/rejected": -1.1202101707458496, + "logps/chosen": -264.85552978515625, + "logps/rejected": -270.9376525878906, + "loss": 0.6927, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.0047036223113536835, + "rewards/margins": 0.0003113269922323525, + "rewards/rejected": -0.005014949012547731, + "step": 15 + }, + { + "epoch": 0.03419716804702111, + "grad_norm": 4.9412589689200095, + "learning_rate": 2.3829787234042553e-07, + "logits/chosen": -1.0979671478271484, + "logits/rejected": -1.067629098892212, + "logps/chosen": -271.14385986328125, + "logps/rejected": -280.5599670410156, + "loss": 0.6922, + "rewards/accuracies": 0.5, + "rewards/chosen": -9.246054105460644e-06, + "rewards/margins": 0.0020150854252278805, + "rewards/rejected": -0.0020243311300873756, + "step": 16 + }, + { + "epoch": 0.03633449104995993, + "grad_norm": 5.374113235339209, + "learning_rate": 2.5319148936170213e-07, + "logits/chosen": -1.1378012895584106, + "logits/rejected": -1.087912678718567, + "logps/chosen": -237.7337188720703, + "logps/rejected": -237.0882110595703, + "loss": 0.6931, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.003059606533497572, + "rewards/margins": -4.628184251487255e-05, + "rewards/rejected": -0.003013324923813343, + "step": 17 + }, + { + "epoch": 0.03847181405289874, + "grad_norm": 6.092267360675494, + "learning_rate": 2.680851063829787e-07, + "logits/chosen": -1.0425890684127808, + "logits/rejected": -1.0599486827850342, + "logps/chosen": -242.11541748046875, + "logps/rejected": -256.46417236328125, + "loss": 0.6928, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.004783968441188335, + "rewards/margins": -9.258274803869426e-05, + "rewards/rejected": -0.004691386129707098, + "step": 18 + }, + { + "epoch": 0.04060913705583756, + "grad_norm": 5.1497648303931784, + "learning_rate": 2.829787234042553e-07, + "logits/chosen": -1.1328908205032349, + "logits/rejected": -1.0197917222976685, + "logps/chosen": -253.92127990722656, + "logps/rejected": -278.0308837890625, + "loss": 0.6923, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.0021464251913130283, + "rewards/margins": 0.00046774852671660483, + "rewards/rejected": -0.0026141738053411245, + "step": 19 + }, + { + "epoch": 0.04274646005877638, + "grad_norm": 5.9119652568805225, + "learning_rate": 2.978723404255319e-07, + "logits/chosen": -1.1393866539001465, + "logits/rejected": -1.0607531070709229, + "logps/chosen": -303.4728088378906, + "logps/rejected": -295.6599426269531, + "loss": 0.6916, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.004760389216244221, + "rewards/margins": 0.003504714695736766, + "rewards/rejected": -0.00826510414481163, + "step": 20 + }, + { + "epoch": 0.0448837830617152, + "grad_norm": 5.752883808298607, + "learning_rate": 3.1276595744680846e-07, + "logits/chosen": -1.1522903442382812, + "logits/rejected": -1.1488419771194458, + "logps/chosen": -192.28042602539062, + "logps/rejected": -200.01608276367188, + "loss": 0.6911, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.004406332969665527, + "rewards/margins": -0.0005414390470832586, + "rewards/rejected": -0.0038648939225822687, + "step": 21 + }, + { + "epoch": 0.04702110606465402, + "grad_norm": 4.856538956353405, + "learning_rate": 3.276595744680851e-07, + "logits/chosen": -1.0699973106384277, + "logits/rejected": -1.0846004486083984, + "logps/chosen": -205.7268829345703, + "logps/rejected": -201.65975952148438, + "loss": 0.6915, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.005930948071181774, + "rewards/margins": 0.0032684658654034138, + "rewards/rejected": -0.009199414402246475, + "step": 22 + }, + { + "epoch": 0.04915842906759284, + "grad_norm": 4.906648390477566, + "learning_rate": 3.425531914893617e-07, + "logits/chosen": -0.9317066669464111, + "logits/rejected": -0.8009700179100037, + "logps/chosen": -246.80438232421875, + "logps/rejected": -272.9093322753906, + "loss": 0.6911, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.010996412485837936, + "rewards/margins": 0.0014129301998764277, + "rewards/rejected": -0.012409343384206295, + "step": 23 + }, + { + "epoch": 0.05129575207053166, + "grad_norm": 5.875935612586806, + "learning_rate": 3.5744680851063824e-07, + "logits/chosen": -1.069819688796997, + "logits/rejected": -0.9709153175354004, + "logps/chosen": -195.9522247314453, + "logps/rejected": -229.61566162109375, + "loss": 0.6909, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.004574608989059925, + "rewards/margins": -0.0003843879676423967, + "rewards/rejected": -0.00419022049754858, + "step": 24 + }, + { + "epoch": 0.053433075073470476, + "grad_norm": 4.985096243829474, + "learning_rate": 3.7234042553191484e-07, + "logits/chosen": -0.8998661637306213, + "logits/rejected": -0.8598207831382751, + "logps/chosen": -282.3105773925781, + "logps/rejected": -272.4259338378906, + "loss": 0.6905, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.010662160813808441, + "rewards/margins": 0.005043415818363428, + "rewards/rejected": -0.015705574303865433, + "step": 25 + }, + { + "epoch": 0.055570398076409296, + "grad_norm": 5.218346832933627, + "learning_rate": 3.872340425531915e-07, + "logits/chosen": -0.9051691889762878, + "logits/rejected": -0.8448677659034729, + "logps/chosen": -241.62689208984375, + "logps/rejected": -257.25244140625, + "loss": 0.6901, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.02060263603925705, + "rewards/margins": -0.0019798323046416044, + "rewards/rejected": -0.018622804433107376, + "step": 26 + }, + { + "epoch": 0.057707721079348115, + "grad_norm": 5.189530040363191, + "learning_rate": 4.021276595744681e-07, + "logits/chosen": -1.0560821294784546, + "logits/rejected": -1.0800153017044067, + "logps/chosen": -252.28952026367188, + "logps/rejected": -245.88502502441406, + "loss": 0.6898, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.015861667692661285, + "rewards/margins": 0.007801623083651066, + "rewards/rejected": -0.023663289844989777, + "step": 27 + }, + { + "epoch": 0.059845044082286934, + "grad_norm": 5.8226698693397, + "learning_rate": 4.170212765957446e-07, + "logits/chosen": -1.0804771184921265, + "logits/rejected": -1.099622130393982, + "logps/chosen": -227.35537719726562, + "logps/rejected": -225.36538696289062, + "loss": 0.6887, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.017352018505334854, + "rewards/margins": 0.01319974847137928, + "rewards/rejected": -0.030551765114068985, + "step": 28 + }, + { + "epoch": 0.061982367085225754, + "grad_norm": 5.460690205724815, + "learning_rate": 4.319148936170213e-07, + "logits/chosen": -1.0153366327285767, + "logits/rejected": -1.0103431940078735, + "logps/chosen": -263.8896789550781, + "logps/rejected": -267.3290100097656, + "loss": 0.6878, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.02432805672287941, + "rewards/margins": 0.008317284286022186, + "rewards/rejected": -0.032645341008901596, + "step": 29 + }, + { + "epoch": 0.06411969008816458, + "grad_norm": 5.737017895955914, + "learning_rate": 4.468085106382979e-07, + "logits/chosen": -1.0464987754821777, + "logits/rejected": -1.1331242322921753, + "logps/chosen": -206.78468322753906, + "logps/rejected": -192.26502990722656, + "loss": 0.6883, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.010598372668027878, + "rewards/margins": 0.002255907515063882, + "rewards/rejected": -0.012854279950261116, + "step": 30 + }, + { + "epoch": 0.06625701309110339, + "grad_norm": 5.795133035841147, + "learning_rate": 4.617021276595744e-07, + "logits/chosen": -0.9297857880592346, + "logits/rejected": -0.8592264652252197, + "logps/chosen": -227.1981964111328, + "logps/rejected": -247.9413299560547, + "loss": 0.6863, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.03003655932843685, + "rewards/margins": 0.009244289249181747, + "rewards/rejected": -0.03928084671497345, + "step": 31 + }, + { + "epoch": 0.06839433609404222, + "grad_norm": 6.814768956718574, + "learning_rate": 4.7659574468085105e-07, + "logits/chosen": -1.05179762840271, + "logits/rejected": -1.0814404487609863, + "logps/chosen": -290.69525146484375, + "logps/rejected": -274.92047119140625, + "loss": 0.6818, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.05112060159444809, + "rewards/margins": 0.015534894540905952, + "rewards/rejected": -0.06665549427270889, + "step": 32 + }, + { + "epoch": 0.07053165909698103, + "grad_norm": 6.074153638196805, + "learning_rate": 4.914893617021277e-07, + "logits/chosen": -0.8849226236343384, + "logits/rejected": -0.9219367504119873, + "logps/chosen": -317.4134826660156, + "logps/rejected": -325.8724365234375, + "loss": 0.6832, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.04802538454532623, + "rewards/margins": 0.024860400706529617, + "rewards/rejected": -0.07288578897714615, + "step": 33 + }, + { + "epoch": 0.07266898209991986, + "grad_norm": 6.0900485696331215, + "learning_rate": 5.063829787234043e-07, + "logits/chosen": -1.0383641719818115, + "logits/rejected": -1.0028424263000488, + "logps/chosen": -277.57965087890625, + "logps/rejected": -268.2386474609375, + "loss": 0.6848, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.060051530599594116, + "rewards/margins": 0.0026317029260098934, + "rewards/rejected": -0.06268322467803955, + "step": 34 + }, + { + "epoch": 0.07480630510285867, + "grad_norm": 6.827504173132289, + "learning_rate": 5.212765957446808e-07, + "logits/chosen": -1.0755884647369385, + "logits/rejected": -1.0121068954467773, + "logps/chosen": -311.99755859375, + "logps/rejected": -350.29052734375, + "loss": 0.682, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.061084531247615814, + "rewards/margins": 0.04312053322792053, + "rewards/rejected": -0.10420507192611694, + "step": 35 + }, + { + "epoch": 0.07694362810579748, + "grad_norm": 6.012937319519413, + "learning_rate": 5.361702127659574e-07, + "logits/chosen": -1.1560208797454834, + "logits/rejected": -1.0447015762329102, + "logps/chosen": -216.53421020507812, + "logps/rejected": -234.53128051757812, + "loss": 0.6788, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.04387297108769417, + "rewards/margins": 0.016591297462582588, + "rewards/rejected": -0.06046426668763161, + "step": 36 + }, + { + "epoch": 0.07908095110873631, + "grad_norm": 5.599688064843719, + "learning_rate": 5.51063829787234e-07, + "logits/chosen": -1.2253515720367432, + "logits/rejected": -1.2332267761230469, + "logps/chosen": -237.71002197265625, + "logps/rejected": -236.66990661621094, + "loss": 0.6799, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.08036091923713684, + "rewards/margins": 0.03013552539050579, + "rewards/rejected": -0.11049643903970718, + "step": 37 + }, + { + "epoch": 0.08121827411167512, + "grad_norm": 6.7746928876503105, + "learning_rate": 5.659574468085106e-07, + "logits/chosen": -1.0830399990081787, + "logits/rejected": -1.0622460842132568, + "logps/chosen": -284.8079833984375, + "logps/rejected": -272.3097839355469, + "loss": 0.6755, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.07364249229431152, + "rewards/margins": 0.03328356519341469, + "rewards/rejected": -0.10692604631185532, + "step": 38 + }, + { + "epoch": 0.08335559711461395, + "grad_norm": 6.77614986840528, + "learning_rate": 5.808510638297872e-07, + "logits/chosen": -1.0905030965805054, + "logits/rejected": -1.0605907440185547, + "logps/chosen": -210.57684326171875, + "logps/rejected": -239.24871826171875, + "loss": 0.6734, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.1003848984837532, + "rewards/margins": 0.024850603193044662, + "rewards/rejected": -0.12523549795150757, + "step": 39 + }, + { + "epoch": 0.08549292011755276, + "grad_norm": 6.424490447178767, + "learning_rate": 5.957446808510638e-07, + "logits/chosen": -1.1801645755767822, + "logits/rejected": -1.1392816305160522, + "logps/chosen": -263.45037841796875, + "logps/rejected": -273.5709533691406, + "loss": 0.6736, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.11433250457048416, + "rewards/margins": 0.028288790956139565, + "rewards/rejected": -0.14262130856513977, + "step": 40 + }, + { + "epoch": 0.08763024312049159, + "grad_norm": 6.4654988266248425, + "learning_rate": 6.106382978723404e-07, + "logits/chosen": -1.0815637111663818, + "logits/rejected": -1.013136625289917, + "logps/chosen": -206.23907470703125, + "logps/rejected": -220.0244140625, + "loss": 0.6722, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.11655673384666443, + "rewards/margins": 0.04582422599196434, + "rewards/rejected": -0.16238096356391907, + "step": 41 + }, + { + "epoch": 0.0897675661234304, + "grad_norm": 6.539055183308233, + "learning_rate": 6.255319148936169e-07, + "logits/chosen": -1.077747106552124, + "logits/rejected": -1.0630099773406982, + "logps/chosen": -268.242431640625, + "logps/rejected": -304.5572204589844, + "loss": 0.6716, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1858682483434677, + "rewards/margins": 0.05748748779296875, + "rewards/rejected": -0.24335573613643646, + "step": 42 + }, + { + "epoch": 0.09190488912636922, + "grad_norm": 6.095364046949072, + "learning_rate": 6.404255319148935e-07, + "logits/chosen": -1.0136995315551758, + "logits/rejected": -0.9615681171417236, + "logps/chosen": -228.80368041992188, + "logps/rejected": -235.71481323242188, + "loss": 0.6689, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.1476023644208908, + "rewards/margins": 0.0479072704911232, + "rewards/rejected": -0.1955096423625946, + "step": 43 + }, + { + "epoch": 0.09404221212930804, + "grad_norm": 6.7344523666741765, + "learning_rate": 6.553191489361702e-07, + "logits/chosen": -0.8863506317138672, + "logits/rejected": -0.9662103652954102, + "logps/chosen": -259.41595458984375, + "logps/rejected": -272.3363037109375, + "loss": 0.6688, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.19808562099933624, + "rewards/margins": 0.013540804386138916, + "rewards/rejected": -0.21162641048431396, + "step": 44 + }, + { + "epoch": 0.09617953513224686, + "grad_norm": 6.29136903272874, + "learning_rate": 6.702127659574469e-07, + "logits/chosen": -1.0862454175949097, + "logits/rejected": -1.03424870967865, + "logps/chosen": -343.55059814453125, + "logps/rejected": -352.5019836425781, + "loss": 0.6619, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.25226354598999023, + "rewards/margins": 0.05790935456752777, + "rewards/rejected": -0.3101729154586792, + "step": 45 + }, + { + "epoch": 0.09831685813518568, + "grad_norm": 5.905042821290849, + "learning_rate": 6.851063829787234e-07, + "logits/chosen": -1.086181402206421, + "logits/rejected": -1.0607441663742065, + "logps/chosen": -281.6853942871094, + "logps/rejected": -306.71136474609375, + "loss": 0.6646, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.30046984553337097, + "rewards/margins": 0.11004211008548737, + "rewards/rejected": -0.41051194071769714, + "step": 46 + }, + { + "epoch": 0.1004541811381245, + "grad_norm": 6.664836667296562, + "learning_rate": 7e-07, + "logits/chosen": -1.1094589233398438, + "logits/rejected": -1.0737080574035645, + "logps/chosen": -276.6365966796875, + "logps/rejected": -290.7496032714844, + "loss": 0.6509, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2919345498085022, + "rewards/margins": 0.12451538443565369, + "rewards/rejected": -0.41644996404647827, + "step": 47 + }, + { + "epoch": 0.10259150414106331, + "grad_norm": 6.452030342745469, + "learning_rate": 6.999902087714444e-07, + "logits/chosen": -1.0068883895874023, + "logits/rejected": -0.955917239189148, + "logps/chosen": -222.2923583984375, + "logps/rejected": -239.01463317871094, + "loss": 0.6577, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1943490207195282, + "rewards/margins": 0.08093887567520142, + "rewards/rejected": -0.275287926197052, + "step": 48 + }, + { + "epoch": 0.10472882714400214, + "grad_norm": 7.754778118919807, + "learning_rate": 6.999608356335957e-07, + "logits/chosen": -1.0677528381347656, + "logits/rejected": -1.0721509456634521, + "logps/chosen": -227.729248046875, + "logps/rejected": -239.76138305664062, + "loss": 0.645, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.2937619090080261, + "rewards/margins": 0.048121094703674316, + "rewards/rejected": -0.3418830335140228, + "step": 49 + }, + { + "epoch": 0.10686615014694095, + "grad_norm": 6.437697145095338, + "learning_rate": 6.999118822298773e-07, + "logits/chosen": -1.1454989910125732, + "logits/rejected": -1.0754855871200562, + "logps/chosen": -227.07974243164062, + "logps/rejected": -228.60333251953125, + "loss": 0.6547, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.2852881848812103, + "rewards/margins": 0.0602235309779644, + "rewards/rejected": -0.3455117344856262, + "step": 50 + }, + { + "epoch": 0.10900347314987978, + "grad_norm": 7.527535438459104, + "learning_rate": 6.998433512992261e-07, + "logits/chosen": -1.1637341976165771, + "logits/rejected": -1.0740972757339478, + "logps/chosen": -292.79742431640625, + "logps/rejected": -302.4303283691406, + "loss": 0.6493, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5110954642295837, + "rewards/margins": 0.09770005941390991, + "rewards/rejected": -0.6087955236434937, + "step": 51 + }, + { + "epoch": 0.11114079615281859, + "grad_norm": 7.575162846371126, + "learning_rate": 6.997552466759395e-07, + "logits/chosen": -1.0683425664901733, + "logits/rejected": -1.1101981401443481, + "logps/chosen": -297.38702392578125, + "logps/rejected": -309.9881286621094, + "loss": 0.6288, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4417276978492737, + "rewards/margins": 0.09606233239173889, + "rewards/rejected": -0.537790060043335, + "step": 52 + }, + { + "epoch": 0.11327811915575742, + "grad_norm": 7.584356018239797, + "learning_rate": 6.996475732894601e-07, + "logits/chosen": -1.0953909158706665, + "logits/rejected": -1.0685151815414429, + "logps/chosen": -336.0571594238281, + "logps/rejected": -369.7088623046875, + "loss": 0.6369, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6481105089187622, + "rewards/margins": 0.13110791146755219, + "rewards/rejected": -0.779218316078186, + "step": 53 + }, + { + "epoch": 0.11541544215869623, + "grad_norm": 6.778514980526427, + "learning_rate": 6.995203371641008e-07, + "logits/chosen": -1.1196974515914917, + "logits/rejected": -1.0348316431045532, + "logps/chosen": -287.28729248046875, + "logps/rejected": -302.94317626953125, + "loss": 0.6444, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5020595788955688, + "rewards/margins": 0.09727220237255096, + "rewards/rejected": -0.5993317365646362, + "step": 54 + }, + { + "epoch": 0.11755276516163506, + "grad_norm": 8.211287073219191, + "learning_rate": 6.993735454187072e-07, + "logits/chosen": -1.0214238166809082, + "logits/rejected": -0.9623696804046631, + "logps/chosen": -265.59417724609375, + "logps/rejected": -298.86444091796875, + "loss": 0.6016, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5154373645782471, + "rewards/margins": 0.24744802713394165, + "rewards/rejected": -0.7628854513168335, + "step": 55 + }, + { + "epoch": 0.11969008816457387, + "grad_norm": 7.2415969346538756, + "learning_rate": 6.992072062662596e-07, + "logits/chosen": -0.9752060174942017, + "logits/rejected": -0.9751684069633484, + "logps/chosen": -349.1664123535156, + "logps/rejected": -362.6107177734375, + "loss": 0.6343, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7859246134757996, + "rewards/margins": 0.13539882004261017, + "rewards/rejected": -0.9213234186172485, + "step": 56 + }, + { + "epoch": 0.1218274111675127, + "grad_norm": 6.635074198180161, + "learning_rate": 6.99021329013413e-07, + "logits/chosen": -1.1370452642440796, + "logits/rejected": -1.0629245042800903, + "logps/chosen": -288.10430908203125, + "logps/rejected": -307.46484375, + "loss": 0.61, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6563497185707092, + "rewards/margins": 0.21564731001853943, + "rewards/rejected": -0.8719969987869263, + "step": 57 + }, + { + "epoch": 0.12396473417045151, + "grad_norm": 7.22705417609884, + "learning_rate": 6.988159240599771e-07, + "logits/chosen": -1.1842763423919678, + "logits/rejected": -1.1428574323654175, + "logps/chosen": -288.5982666015625, + "logps/rejected": -291.8437194824219, + "loss": 0.6186, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7895088195800781, + "rewards/margins": 0.07229967415332794, + "rewards/rejected": -0.8618084788322449, + "step": 58 + }, + { + "epoch": 0.12610205717339032, + "grad_norm": 7.191134813555289, + "learning_rate": 6.985910028983336e-07, + "logits/chosen": -1.0885567665100098, + "logits/rejected": -1.0549125671386719, + "logps/chosen": -484.6271667480469, + "logps/rejected": -517.6636352539062, + "loss": 0.6022, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.1380163431167603, + "rewards/margins": 0.33240166306495667, + "rewards/rejected": -1.4704179763793945, + "step": 59 + }, + { + "epoch": 0.12823938017632916, + "grad_norm": 6.856697802256405, + "learning_rate": 6.983465781127942e-07, + "logits/chosen": -1.0434743165969849, + "logits/rejected": -1.0580846071243286, + "logps/chosen": -315.3133239746094, + "logps/rejected": -314.5843811035156, + "loss": 0.6142, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7601010799407959, + "rewards/margins": 0.006043531000614166, + "rewards/rejected": -0.7661447525024414, + "step": 60 + }, + { + "epoch": 0.13037670317926797, + "grad_norm": 6.8390931208538985, + "learning_rate": 6.980826633788957e-07, + "logits/chosen": -1.0378106832504272, + "logits/rejected": -0.9883963465690613, + "logps/chosen": -406.2007751464844, + "logps/rejected": -471.77471923828125, + "loss": 0.611, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.0759634971618652, + "rewards/margins": 0.3877870440483093, + "rewards/rejected": -1.4637506008148193, + "step": 61 + }, + { + "epoch": 0.13251402618220678, + "grad_norm": 8.256162255166101, + "learning_rate": 6.977992734626349e-07, + "logits/chosen": -1.2170382738113403, + "logits/rejected": -1.1635799407958984, + "logps/chosen": -409.862548828125, + "logps/rejected": -459.107421875, + "loss": 0.5815, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.3327953815460205, + "rewards/margins": 0.5586321353912354, + "rewards/rejected": -1.8914276361465454, + "step": 62 + }, + { + "epoch": 0.1346513491851456, + "grad_norm": 6.7905205812341185, + "learning_rate": 6.97496424219643e-07, + "logits/chosen": -0.9324991106987, + "logits/rejected": -0.9242455959320068, + "logps/chosen": -305.8074035644531, + "logps/rejected": -359.38275146484375, + "loss": 0.5897, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.7552638649940491, + "rewards/margins": 0.3947301506996155, + "rewards/rejected": -1.1499940156936646, + "step": 63 + }, + { + "epoch": 0.13678867218808444, + "grad_norm": 7.929159551201501, + "learning_rate": 6.971741325942982e-07, + "logits/chosen": -1.0476861000061035, + "logits/rejected": -1.0574254989624023, + "logps/chosen": -397.0540466308594, + "logps/rejected": -445.9757385253906, + "loss": 0.5989, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.1025577783584595, + "rewards/margins": 0.5505799055099487, + "rewards/rejected": -1.6531375646591187, + "step": 64 + }, + { + "epoch": 0.13892599519102325, + "grad_norm": 6.60300962363944, + "learning_rate": 6.968324166187772e-07, + "logits/chosen": -1.0173978805541992, + "logits/rejected": -1.0060231685638428, + "logps/chosen": -391.6329345703125, + "logps/rejected": -482.0704040527344, + "loss": 0.5949, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.2589941024780273, + "rewards/margins": 0.7740709781646729, + "rewards/rejected": -2.0330650806427, + "step": 65 + }, + { + "epoch": 0.14106331819396206, + "grad_norm": 7.722534638937169, + "learning_rate": 6.96471295412047e-07, + "logits/chosen": -1.210124135017395, + "logits/rejected": -1.1669820547103882, + "logps/chosen": -436.8033447265625, + "logps/rejected": -447.0545349121094, + "loss": 0.6197, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.7195076942443848, + "rewards/margins": 0.12996703386306763, + "rewards/rejected": -1.8494747877120972, + "step": 66 + }, + { + "epoch": 0.14320064119690087, + "grad_norm": 8.147380246018754, + "learning_rate": 6.960907891787949e-07, + "logits/chosen": -1.165956974029541, + "logits/rejected": -1.1428234577178955, + "logps/chosen": -519.3298950195312, + "logps/rejected": -559.141357421875, + "loss": 0.547, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.083847999572754, + "rewards/margins": 0.4430221617221832, + "rewards/rejected": -2.5268702507019043, + "step": 67 + }, + { + "epoch": 0.14533796419983971, + "grad_norm": 7.8944938583131945, + "learning_rate": 6.956909192082982e-07, + "logits/chosen": -0.9343075752258301, + "logits/rejected": -0.8972913026809692, + "logps/chosen": -292.84552001953125, + "logps/rejected": -341.25872802734375, + "loss": 0.5741, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.985328197479248, + "rewards/margins": 0.39189204573631287, + "rewards/rejected": -1.3772202730178833, + "step": 68 + }, + { + "epoch": 0.14747528720277853, + "grad_norm": 7.019821529750172, + "learning_rate": 6.952717078732324e-07, + "logits/chosen": -1.0702323913574219, + "logits/rejected": -0.9579718112945557, + "logps/chosen": -378.2999572753906, + "logps/rejected": -452.3642578125, + "loss": 0.6007, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.3012688159942627, + "rewards/margins": 0.5469968318939209, + "rewards/rejected": -1.8482656478881836, + "step": 69 + }, + { + "epoch": 0.14961261020571734, + "grad_norm": 7.623144170728904, + "learning_rate": 6.948331786284207e-07, + "logits/chosen": -1.0306965112686157, + "logits/rejected": -1.0405011177062988, + "logps/chosen": -489.2743835449219, + "logps/rejected": -580.343994140625, + "loss": 0.5627, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.306447982788086, + "rewards/margins": 0.7942487001419067, + "rewards/rejected": -3.100696563720703, + "step": 70 + }, + { + "epoch": 0.15174993320865615, + "grad_norm": 7.340408773148345, + "learning_rate": 6.943753560095204e-07, + "logits/chosen": -1.1813827753067017, + "logits/rejected": -1.1872248649597168, + "logps/chosen": -516.26953125, + "logps/rejected": -557.9083251953125, + "loss": 0.563, + "rewards/accuracies": 0.5625, + "rewards/chosen": -2.189997434616089, + "rewards/margins": 0.49108394980430603, + "rewards/rejected": -2.681081533432007, + "step": 71 + }, + { + "epoch": 0.15388725621159496, + "grad_norm": 9.09254480811679, + "learning_rate": 6.938982656316509e-07, + "logits/chosen": -1.108089566230774, + "logits/rejected": -1.1164076328277588, + "logps/chosen": -483.02099609375, + "logps/rejected": -499.59503173828125, + "loss": 0.5837, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.988714575767517, + "rewards/margins": 0.3150157928466797, + "rewards/rejected": -2.3037304878234863, + "step": 72 + }, + { + "epoch": 0.1560245792145338, + "grad_norm": 10.656479693855966, + "learning_rate": 6.934019341879603e-07, + "logits/chosen": -0.9998635649681091, + "logits/rejected": -0.9844257831573486, + "logps/chosen": -449.13592529296875, + "logps/rejected": -543.6675415039062, + "loss": 0.6031, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9461640119552612, + "rewards/margins": 0.7130802869796753, + "rewards/rejected": -2.6592442989349365, + "step": 73 + }, + { + "epoch": 0.15816190221747262, + "grad_norm": 10.135891828278513, + "learning_rate": 6.928863894481325e-07, + "logits/chosen": -1.057843804359436, + "logits/rejected": -1.0019515752792358, + "logps/chosen": -499.46343994140625, + "logps/rejected": -607.9090576171875, + "loss": 0.6289, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.925887942314148, + "rewards/margins": 0.7596069574356079, + "rewards/rejected": -2.685494899749756, + "step": 74 + }, + { + "epoch": 0.16029922522041143, + "grad_norm": 11.196785279167383, + "learning_rate": 6.923516602568319e-07, + "logits/chosen": -1.10731840133667, + "logits/rejected": -1.1392724514007568, + "logps/chosen": -572.486083984375, + "logps/rejected": -581.3994140625, + "loss": 0.6307, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.843285322189331, + "rewards/margins": 0.3052109479904175, + "rewards/rejected": -3.148496150970459, + "step": 75 + }, + { + "epoch": 0.16243654822335024, + "grad_norm": 8.037071469258775, + "learning_rate": 6.917977765320918e-07, + "logits/chosen": -1.0746040344238281, + "logits/rejected": -1.0597764253616333, + "logps/chosen": -504.6460266113281, + "logps/rejected": -606.507080078125, + "loss": 0.5222, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.3728506565093994, + "rewards/margins": 0.8532505035400391, + "rewards/rejected": -3.2261009216308594, + "step": 76 + }, + { + "epoch": 0.16457387122628908, + "grad_norm": 8.046205826147432, + "learning_rate": 6.912247692636382e-07, + "logits/chosen": -0.9819946885108948, + "logits/rejected": -0.9038519859313965, + "logps/chosen": -402.05523681640625, + "logps/rejected": -466.16973876953125, + "loss": 0.5378, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7473161220550537, + "rewards/margins": 0.5481730699539185, + "rewards/rejected": -2.2954893112182617, + "step": 77 + }, + { + "epoch": 0.1667111942292279, + "grad_norm": 8.442110303754333, + "learning_rate": 6.90632670511158e-07, + "logits/chosen": -1.0492621660232544, + "logits/rejected": -1.0028660297393799, + "logps/chosen": -452.12823486328125, + "logps/rejected": -512.2908935546875, + "loss": 0.5626, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.2684264183044434, + "rewards/margins": 0.5885436534881592, + "rewards/rejected": -2.8569703102111816, + "step": 78 + }, + { + "epoch": 0.1688485172321667, + "grad_norm": 8.834706933620803, + "learning_rate": 6.900215134025036e-07, + "logits/chosen": -0.9072580933570862, + "logits/rejected": -0.896135687828064, + "logps/chosen": -446.088134765625, + "logps/rejected": -527.0875854492188, + "loss": 0.5476, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0337717533111572, + "rewards/margins": 0.5865947604179382, + "rewards/rejected": -2.6203665733337402, + "step": 79 + }, + { + "epoch": 0.17098584023510552, + "grad_norm": 8.951583471345572, + "learning_rate": 6.893913321318404e-07, + "logits/chosen": -0.9730253219604492, + "logits/rejected": -0.988685131072998, + "logps/chosen": -460.3742370605469, + "logps/rejected": -495.2950134277344, + "loss": 0.5833, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.2925660610198975, + "rewards/margins": 0.45126643776893616, + "rewards/rejected": -2.743832588195801, + "step": 80 + }, + { + "epoch": 0.17312316323804436, + "grad_norm": 10.31156332894422, + "learning_rate": 6.887421619577333e-07, + "logits/chosen": -0.9978621602058411, + "logits/rejected": -0.914361298084259, + "logps/chosen": -577.339111328125, + "logps/rejected": -620.412353515625, + "loss": 0.5333, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.5646235942840576, + "rewards/margins": 0.5104097127914429, + "rewards/rejected": -3.075033187866211, + "step": 81 + }, + { + "epoch": 0.17526048624098317, + "grad_norm": 8.242407615983574, + "learning_rate": 6.880740392011738e-07, + "logits/chosen": -1.0842255353927612, + "logits/rejected": -1.0887690782546997, + "logps/chosen": -484.07733154296875, + "logps/rejected": -524.2720947265625, + "loss": 0.5808, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.4766807556152344, + "rewards/margins": 0.48678654432296753, + "rewards/rejected": -2.963467597961426, + "step": 82 + }, + { + "epoch": 0.17739780924392198, + "grad_norm": 9.401026195070148, + "learning_rate": 6.873870012435486e-07, + "logits/chosen": -1.0653420686721802, + "logits/rejected": -1.0636709928512573, + "logps/chosen": -476.5955810546875, + "logps/rejected": -574.3685302734375, + "loss": 0.5926, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.0014445781707764, + "rewards/margins": 1.0705955028533936, + "rewards/rejected": -3.072040319442749, + "step": 83 + }, + { + "epoch": 0.1795351322468608, + "grad_norm": 9.603090833181401, + "learning_rate": 6.866810865245471e-07, + "logits/chosen": -1.1408371925354004, + "logits/rejected": -1.2083356380462646, + "logps/chosen": -481.21270751953125, + "logps/rejected": -499.5158386230469, + "loss": 0.5634, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.340351104736328, + "rewards/margins": 0.2937456965446472, + "rewards/rejected": -2.634096622467041, + "step": 84 + }, + { + "epoch": 0.18167245524979964, + "grad_norm": 8.394856753842978, + "learning_rate": 6.859563345400111e-07, + "logits/chosen": -1.0628814697265625, + "logits/rejected": -0.9136342406272888, + "logps/chosen": -399.5640869140625, + "logps/rejected": -533.1271362304688, + "loss": 0.5817, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9623302221298218, + "rewards/margins": 1.1472116708755493, + "rewards/rejected": -3.109541893005371, + "step": 85 + }, + { + "epoch": 0.18380977825273845, + "grad_norm": 8.518782849174428, + "learning_rate": 6.852127858397256e-07, + "logits/chosen": -1.066598892211914, + "logits/rejected": -1.0352439880371094, + "logps/chosen": -561.7534790039062, + "logps/rejected": -711.9557495117188, + "loss": 0.521, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.871816396713257, + "rewards/margins": 1.3357863426208496, + "rewards/rejected": -4.207602500915527, + "step": 86 + }, + { + "epoch": 0.18594710125567726, + "grad_norm": 9.531847125052328, + "learning_rate": 6.844504820251493e-07, + "logits/chosen": -0.8711273670196533, + "logits/rejected": -0.7599645853042603, + "logps/chosen": -630.6957397460938, + "logps/rejected": -768.0391845703125, + "loss": 0.5164, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.074097156524658, + "rewards/margins": 1.3850178718566895, + "rewards/rejected": -4.459115028381348, + "step": 87 + }, + { + "epoch": 0.18808442425861607, + "grad_norm": 11.193362121063247, + "learning_rate": 6.836694657470869e-07, + "logits/chosen": -1.0319950580596924, + "logits/rejected": -0.9569318294525146, + "logps/chosen": -283.17431640625, + "logps/rejected": -333.98797607421875, + "loss": 0.5972, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.3249003887176514, + "rewards/margins": 0.47719448804855347, + "rewards/rejected": -1.8020949363708496, + "step": 88 + }, + { + "epoch": 0.1902217472615549, + "grad_norm": 12.5393990232581, + "learning_rate": 6.828697807033038e-07, + "logits/chosen": -1.016239881515503, + "logits/rejected": -0.9772812724113464, + "logps/chosen": -387.52056884765625, + "logps/rejected": -477.847412109375, + "loss": 0.5869, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.845961332321167, + "rewards/margins": 0.6933265328407288, + "rewards/rejected": -2.539287805557251, + "step": 89 + }, + { + "epoch": 0.19235907026449373, + "grad_norm": 11.082808377296098, + "learning_rate": 6.820514716360799e-07, + "logits/chosen": -0.9989445209503174, + "logits/rejected": -1.0149664878845215, + "logps/chosen": -550.3873291015625, + "logps/rejected": -627.486328125, + "loss": 0.5262, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.6533279418945312, + "rewards/margins": 0.7726477980613708, + "rewards/rejected": -3.425976037979126, + "step": 90 + }, + { + "epoch": 0.19449639326743254, + "grad_norm": 10.136450815566235, + "learning_rate": 6.812145843297075e-07, + "logits/chosen": -0.9787087440490723, + "logits/rejected": -0.9385868310928345, + "logps/chosen": -541.9707641601562, + "logps/rejected": -582.03759765625, + "loss": 0.537, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.860710382461548, + "rewards/margins": 0.4204621911048889, + "rewards/rejected": -3.281172513961792, + "step": 91 + }, + { + "epoch": 0.19663371627037135, + "grad_norm": 9.876929179683522, + "learning_rate": 6.803591656079286e-07, + "logits/chosen": -1.1552644968032837, + "logits/rejected": -1.1622107028961182, + "logps/chosen": -502.1416015625, + "logps/rejected": -571.0797729492188, + "loss": 0.5405, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.547774076461792, + "rewards/margins": 0.6767189502716064, + "rewards/rejected": -3.2244930267333984, + "step": 92 + }, + { + "epoch": 0.1987710392733102, + "grad_norm": 8.350168280012312, + "learning_rate": 6.794852633313161e-07, + "logits/chosen": -0.8592644333839417, + "logits/rejected": -0.8664505481719971, + "logps/chosen": -385.8682556152344, + "logps/rejected": -391.97564697265625, + "loss": 0.5449, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.7356358766555786, + "rewards/margins": 0.2472929060459137, + "rewards/rejected": -1.9829288721084595, + "step": 93 + }, + { + "epoch": 0.200908362276249, + "grad_norm": 10.381869975048419, + "learning_rate": 6.785929263945951e-07, + "logits/chosen": -0.9588985443115234, + "logits/rejected": -0.982033371925354, + "logps/chosen": -562.7421875, + "logps/rejected": -626.1223754882812, + "loss": 0.5568, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.8523409366607666, + "rewards/margins": 0.6437615752220154, + "rewards/rejected": -3.4961023330688477, + "step": 94 + }, + { + "epoch": 0.20304568527918782, + "grad_norm": 8.994516081218443, + "learning_rate": 6.776822047239079e-07, + "logits/chosen": -1.0020171403884888, + "logits/rejected": -0.9751790761947632, + "logps/chosen": -417.7941589355469, + "logps/rejected": -482.18511962890625, + "loss": 0.5187, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.281278610229492, + "rewards/margins": 0.5854904651641846, + "rewards/rejected": -2.866769313812256, + "step": 95 + }, + { + "epoch": 0.20518300828212663, + "grad_norm": 9.433844383462462, + "learning_rate": 6.767531492740206e-07, + "logits/chosen": -1.1744908094406128, + "logits/rejected": -1.152523159980774, + "logps/chosen": -517.0827026367188, + "logps/rejected": -538.0679931640625, + "loss": 0.5032, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9695494174957275, + "rewards/margins": 0.5735199451446533, + "rewards/rejected": -2.543069362640381, + "step": 96 + }, + { + "epoch": 0.20732033128506547, + "grad_norm": 12.358671729964394, + "learning_rate": 6.758058120254714e-07, + "logits/chosen": -1.0569499731063843, + "logits/rejected": -0.9990131855010986, + "logps/chosen": -499.7625427246094, + "logps/rejected": -545.9909057617188, + "loss": 0.5433, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.38741397857666, + "rewards/margins": 0.40309280157089233, + "rewards/rejected": -2.7905068397521973, + "step": 97 + }, + { + "epoch": 0.20945765428800428, + "grad_norm": 13.359908029714509, + "learning_rate": 6.748402459816636e-07, + "logits/chosen": -0.9123142957687378, + "logits/rejected": -0.9710780382156372, + "logps/chosen": -639.8603515625, + "logps/rejected": -638.2657470703125, + "loss": 0.5369, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.3181374073028564, + "rewards/margins": 0.10219268500804901, + "rewards/rejected": -3.420330286026001, + "step": 98 + }, + { + "epoch": 0.2115949772909431, + "grad_norm": 9.360941415209796, + "learning_rate": 6.738565051658988e-07, + "logits/chosen": -0.9268811941146851, + "logits/rejected": -1.0143418312072754, + "logps/chosen": -504.1163330078125, + "logps/rejected": -515.728271484375, + "loss": 0.5485, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.5163679122924805, + "rewards/margins": 0.32084643840789795, + "rewards/rejected": -2.837214469909668, + "step": 99 + }, + { + "epoch": 0.2137323002938819, + "grad_norm": 10.094138733652892, + "learning_rate": 6.728546446183552e-07, + "logits/chosen": -0.6902378797531128, + "logits/rejected": -0.7625846266746521, + "logps/chosen": -370.24859619140625, + "logps/rejected": -434.644287109375, + "loss": 0.5813, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.854517936706543, + "rewards/margins": 0.39516037702560425, + "rewards/rejected": -2.249678373336792, + "step": 100 + }, + { + "epoch": 0.2137323002938819, + "eval_logits/chosen": -1.1082046031951904, + "eval_logits/rejected": -1.1243932247161865, + "eval_logps/chosen": -541.2933349609375, + "eval_logps/rejected": -612.4234008789062, + "eval_loss": 0.5237933397293091, + "eval_rewards/accuracies": 0.7298387289047241, + "eval_rewards/chosen": -2.681633710861206, + "eval_rewards/margins": 0.7722625732421875, + "eval_rewards/rejected": -3.4538962841033936, + "eval_runtime": 141.8383, + "eval_samples_per_second": 13.826, + "eval_steps_per_second": 0.437, + "step": 100 + }, + { + "epoch": 0.21586962329682075, + "grad_norm": 13.595024692594054, + "learning_rate": 6.718347203930077e-07, + "logits/chosen": -1.0973420143127441, + "logits/rejected": -1.1126675605773926, + "logps/chosen": -547.3043823242188, + "logps/rejected": -647.8222045898438, + "loss": 0.6014, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.594785690307617, + "rewards/margins": 1.0374321937561035, + "rewards/rejected": -3.6322176456451416, + "step": 101 + }, + { + "epoch": 0.21800694629975956, + "grad_norm": 10.896734874060659, + "learning_rate": 6.707967895544916e-07, + "logits/chosen": -1.0957213640213013, + "logits/rejected": -0.958062469959259, + "logps/chosen": -466.9610595703125, + "logps/rejected": -579.3587036132812, + "loss": 0.4572, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.131477117538452, + "rewards/margins": 0.9194568395614624, + "rewards/rejected": -3.050933837890625, + "step": 102 + }, + { + "epoch": 0.22014426930269837, + "grad_norm": 7.969619734080228, + "learning_rate": 6.697409101749103e-07, + "logits/chosen": -0.9213029146194458, + "logits/rejected": -0.8857940435409546, + "logps/chosen": -506.6609191894531, + "logps/rejected": -597.8681640625, + "loss": 0.4681, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.1986255645751953, + "rewards/margins": 0.973738431930542, + "rewards/rejected": -3.172363758087158, + "step": 103 + }, + { + "epoch": 0.22228159230563718, + "grad_norm": 8.749688164160379, + "learning_rate": 6.686671413305856e-07, + "logits/chosen": -1.1210659742355347, + "logits/rejected": -1.0568785667419434, + "logps/chosen": -527.4649658203125, + "logps/rejected": -666.8826904296875, + "loss": 0.4962, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.895803213119507, + "rewards/margins": 1.2766165733337402, + "rewards/rejected": -4.172420024871826, + "step": 104 + }, + { + "epoch": 0.224418915308576, + "grad_norm": 10.816785495540321, + "learning_rate": 6.675755430987527e-07, + "logits/chosen": -1.098203420639038, + "logits/rejected": -1.043796181678772, + "logps/chosen": -406.22186279296875, + "logps/rejected": -515.1035766601562, + "loss": 0.5235, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.0460591316223145, + "rewards/margins": 0.9124148488044739, + "rewards/rejected": -2.9584736824035645, + "step": 105 + }, + { + "epoch": 0.22655623831151483, + "grad_norm": 16.006357368038902, + "learning_rate": 6.66466176554199e-07, + "logits/chosen": -1.1074966192245483, + "logits/rejected": -1.1250437498092651, + "logps/chosen": -511.63519287109375, + "logps/rejected": -583.53759765625, + "loss": 0.6286, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.4097325801849365, + "rewards/margins": 0.7932775020599365, + "rewards/rejected": -3.203009843826294, + "step": 106 + }, + { + "epoch": 0.22869356131445365, + "grad_norm": 10.145999208049496, + "learning_rate": 6.653391037658466e-07, + "logits/chosen": -0.8575969934463501, + "logits/rejected": -0.8596129417419434, + "logps/chosen": -600.901123046875, + "logps/rejected": -702.7192993164062, + "loss": 0.4988, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.2228050231933594, + "rewards/margins": 0.8592959642410278, + "rewards/rejected": -4.082100868225098, + "step": 107 + }, + { + "epoch": 0.23083088431739246, + "grad_norm": 9.90205874296923, + "learning_rate": 6.641943877932801e-07, + "logits/chosen": -0.982778787612915, + "logits/rejected": -0.967957615852356, + "logps/chosen": -637.15478515625, + "logps/rejected": -736.35546875, + "loss": 0.5006, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.472803831100464, + "rewards/margins": 0.9627398252487183, + "rewards/rejected": -4.435543537139893, + "step": 108 + }, + { + "epoch": 0.23296820732033127, + "grad_norm": 10.737852627360994, + "learning_rate": 6.630320926832178e-07, + "logits/chosen": -1.0073411464691162, + "logits/rejected": -0.9530993700027466, + "logps/chosen": -543.8530883789062, + "logps/rejected": -669.4638061523438, + "loss": 0.5518, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.8368093967437744, + "rewards/margins": 1.1880838871002197, + "rewards/rejected": -4.024893283843994, + "step": 109 + }, + { + "epoch": 0.2351055303232701, + "grad_norm": 10.25518470798023, + "learning_rate": 6.618522834659287e-07, + "logits/chosen": -1.0670475959777832, + "logits/rejected": -0.9968966245651245, + "logps/chosen": -551.1478271484375, + "logps/rejected": -710.794921875, + "loss": 0.4774, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.1101412773132324, + "rewards/margins": 1.3935075998306274, + "rewards/rejected": -4.50364875793457, + "step": 110 + }, + { + "epoch": 0.23724285332620892, + "grad_norm": 10.88392855187359, + "learning_rate": 6.606550261515939e-07, + "logits/chosen": -1.1226112842559814, + "logits/rejected": -1.1211527585983276, + "logps/chosen": -496.4976806640625, + "logps/rejected": -598.9983520507812, + "loss": 0.5214, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.4094114303588867, + "rewards/margins": 0.967483639717102, + "rewards/rejected": -3.37689471244812, + "step": 111 + }, + { + "epoch": 0.23938017632914774, + "grad_norm": 14.5461852961629, + "learning_rate": 6.594403877266134e-07, + "logits/chosen": -0.9625245332717896, + "logits/rejected": -1.0075706243515015, + "logps/chosen": -479.2593688964844, + "logps/rejected": -569.9885864257812, + "loss": 0.5683, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.5545759201049805, + "rewards/margins": 0.8655453324317932, + "rewards/rejected": -3.420121431350708, + "step": 112 + }, + { + "epoch": 0.24151749933208655, + "grad_norm": 10.257155762835067, + "learning_rate": 6.582084361498582e-07, + "logits/chosen": -1.0782970190048218, + "logits/rejected": -1.0272691249847412, + "logps/chosen": -431.1832580566406, + "logps/rejected": -507.42401123046875, + "loss": 0.4517, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.3007423877716064, + "rewards/margins": 0.6344060301780701, + "rewards/rejected": -2.935148239135742, + "step": 113 + }, + { + "epoch": 0.2436548223350254, + "grad_norm": 12.281013811232151, + "learning_rate": 6.569592403488683e-07, + "logits/chosen": -0.8818793296813965, + "logits/rejected": -0.8034133315086365, + "logps/chosen": -575.656005859375, + "logps/rejected": -665.9652709960938, + "loss": 0.5607, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.9418675899505615, + "rewards/margins": 0.8054768443107605, + "rewards/rejected": -3.747344493865967, + "step": 114 + }, + { + "epoch": 0.2457921453379642, + "grad_norm": 12.328703519510336, + "learning_rate": 6.556928702159954e-07, + "logits/chosen": -1.0138773918151855, + "logits/rejected": -0.9439175128936768, + "logps/chosen": -389.8624267578125, + "logps/rejected": -458.929443359375, + "loss": 0.5415, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.944897174835205, + "rewards/margins": 0.5071231126785278, + "rewards/rejected": -2.4520201683044434, + "step": 115 + }, + { + "epoch": 0.24792946834090301, + "grad_norm": 11.019958842452935, + "learning_rate": 6.544093966044933e-07, + "logits/chosen": -1.0471012592315674, + "logits/rejected": -1.046525001525879, + "logps/chosen": -589.4481811523438, + "logps/rejected": -666.8422241210938, + "loss": 0.4744, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.920729160308838, + "rewards/margins": 0.9037714004516602, + "rewards/rejected": -3.824500560760498, + "step": 116 + }, + { + "epoch": 0.25006679134384185, + "grad_norm": 12.320353833685262, + "learning_rate": 6.531088913245536e-07, + "logits/chosen": -0.9775707721710205, + "logits/rejected": -1.0437175035476685, + "logps/chosen": -559.2511596679688, + "logps/rejected": -629.9223022460938, + "loss": 0.5403, + "rewards/accuracies": 0.5625, + "rewards/chosen": -2.863795280456543, + "rewards/margins": 0.9548764228820801, + "rewards/rejected": -3.818671703338623, + "step": 117 + }, + { + "epoch": 0.25220411434678064, + "grad_norm": 13.023997265472861, + "learning_rate": 6.517914271392868e-07, + "logits/chosen": -0.9753481149673462, + "logits/rejected": -0.9616096019744873, + "logps/chosen": -393.8183288574219, + "logps/rejected": -467.1902770996094, + "loss": 0.5117, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.0589523315429688, + "rewards/margins": 0.6962214708328247, + "rewards/rejected": -2.755174160003662, + "step": 118 + }, + { + "epoch": 0.2543414373497195, + "grad_norm": 10.577094903989462, + "learning_rate": 6.504570777606531e-07, + "logits/chosen": -1.1235861778259277, + "logits/rejected": -1.0339062213897705, + "logps/chosen": -567.0230712890625, + "logps/rejected": -711.5052490234375, + "loss": 0.5027, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.0050158500671387, + "rewards/margins": 1.1521648168563843, + "rewards/rejected": -4.1571807861328125, + "step": 119 + }, + { + "epoch": 0.2564787603526583, + "grad_norm": 10.49693020043088, + "learning_rate": 6.491059178453366e-07, + "logits/chosen": -0.9402109980583191, + "logits/rejected": -0.9911918044090271, + "logps/chosen": -524.8615112304688, + "logps/rejected": -612.2509765625, + "loss": 0.4693, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.6408395767211914, + "rewards/margins": 0.9583368301391602, + "rewards/rejected": -3.5991761684417725, + "step": 120 + }, + { + "epoch": 0.2586160833555971, + "grad_norm": 11.423558750572097, + "learning_rate": 6.477380229905688e-07, + "logits/chosen": -1.072409987449646, + "logits/rejected": -1.1194437742233276, + "logps/chosen": -561.1153564453125, + "logps/rejected": -709.1156616210938, + "loss": 0.4927, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.7978341579437256, + "rewards/margins": 1.542366862297058, + "rewards/rejected": -4.340201377868652, + "step": 121 + }, + { + "epoch": 0.26075340635853594, + "grad_norm": 11.084946490602835, + "learning_rate": 6.463534697298995e-07, + "logits/chosen": -0.9931835532188416, + "logits/rejected": -0.8609973788261414, + "logps/chosen": -422.6210632324219, + "logps/rejected": -529.1079711914062, + "loss": 0.4607, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9958739280700684, + "rewards/margins": 0.8684347867965698, + "rewards/rejected": -2.8643088340759277, + "step": 122 + }, + { + "epoch": 0.26289072936147473, + "grad_norm": 12.662380910929564, + "learning_rate": 6.449523355289137e-07, + "logits/chosen": -1.0383917093276978, + "logits/rejected": -1.0384039878845215, + "logps/chosen": -552.278076171875, + "logps/rejected": -629.6370849609375, + "loss": 0.5599, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.6315407752990723, + "rewards/margins": 0.8207972049713135, + "rewards/rejected": -3.4523377418518066, + "step": 123 + }, + { + "epoch": 0.26502805236441357, + "grad_norm": 12.221280650614627, + "learning_rate": 6.435346987808983e-07, + "logits/chosen": -1.0482319593429565, + "logits/rejected": -1.0103380680084229, + "logps/chosen": -566.80078125, + "logps/rejected": -658.3344116210938, + "loss": 0.5172, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.72255539894104, + "rewards/margins": 1.0078556537628174, + "rewards/rejected": -3.7304112911224365, + "step": 124 + }, + { + "epoch": 0.2671653753673524, + "grad_norm": 10.864584700405295, + "learning_rate": 6.421006388024559e-07, + "logits/chosen": -0.9935693740844727, + "logits/rejected": -1.0035309791564941, + "logps/chosen": -450.3325500488281, + "logps/rejected": -518.1874389648438, + "loss": 0.4811, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.3707103729248047, + "rewards/margins": 0.6181319952011108, + "rewards/rejected": -2.988842248916626, + "step": 125 + }, + { + "epoch": 0.2693026983702912, + "grad_norm": 12.045454242902174, + "learning_rate": 6.406502358290663e-07, + "logits/chosen": -1.1015965938568115, + "logits/rejected": -1.08380126953125, + "logps/chosen": -496.258056640625, + "logps/rejected": -653.9327392578125, + "loss": 0.5494, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.305727481842041, + "rewards/margins": 1.4904249906539917, + "rewards/rejected": -3.796152353286743, + "step": 126 + }, + { + "epoch": 0.27144002137323003, + "grad_norm": 18.364703541342568, + "learning_rate": 6.391835710105981e-07, + "logits/chosen": -0.941745400428772, + "logits/rejected": -0.9719420671463013, + "logps/chosen": -511.3363952636719, + "logps/rejected": -570.1834716796875, + "loss": 0.5391, + "rewards/accuracies": 0.5625, + "rewards/chosen": -2.4952189922332764, + "rewards/margins": 0.8485791683197021, + "rewards/rejected": -3.3437979221343994, + "step": 127 + }, + { + "epoch": 0.2735773443761689, + "grad_norm": 11.918550043243753, + "learning_rate": 6.377007264067684e-07, + "logits/chosen": -0.9891614317893982, + "logits/rejected": -0.9654213190078735, + "logps/chosen": -455.00775146484375, + "logps/rejected": -501.5223083496094, + "loss": 0.5492, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.259676218032837, + "rewards/margins": 0.5743985176086426, + "rewards/rejected": -2.8340747356414795, + "step": 128 + }, + { + "epoch": 0.27571466737910766, + "grad_norm": 15.016960436290145, + "learning_rate": 6.36201784982551e-07, + "logits/chosen": -0.9479376673698425, + "logits/rejected": -1.0105921030044556, + "logps/chosen": -484.4723815917969, + "logps/rejected": -579.9881591796875, + "loss": 0.4852, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.237053871154785, + "rewards/margins": 1.103590488433838, + "rewards/rejected": -3.340644121170044, + "step": 129 + }, + { + "epoch": 0.2778519903820465, + "grad_norm": 10.336460981894195, + "learning_rate": 6.346868306035347e-07, + "logits/chosen": -1.0786710977554321, + "logits/rejected": -1.0136293172836304, + "logps/chosen": -531.4271850585938, + "logps/rejected": -632.714599609375, + "loss": 0.4997, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.405646562576294, + "rewards/margins": 1.1792259216308594, + "rewards/rejected": -3.5848724842071533, + "step": 130 + }, + { + "epoch": 0.2799893133849853, + "grad_norm": 13.583887221626354, + "learning_rate": 6.331559480312316e-07, + "logits/chosen": -0.8798739910125732, + "logits/rejected": -0.9477823972702026, + "logps/chosen": -596.0357055664062, + "logps/rejected": -668.8670654296875, + "loss": 0.4991, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.9465270042419434, + "rewards/margins": 0.8481062054634094, + "rewards/rejected": -3.794633150100708, + "step": 131 + }, + { + "epoch": 0.2821266363879241, + "grad_norm": 14.394323156381558, + "learning_rate": 6.316092229183339e-07, + "logits/chosen": -0.9489572048187256, + "logits/rejected": -0.9146859645843506, + "logps/chosen": -600.4107666015625, + "logps/rejected": -661.3778076171875, + "loss": 0.5529, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.2759668827056885, + "rewards/margins": 0.545818567276001, + "rewards/rejected": -3.8217854499816895, + "step": 132 + }, + { + "epoch": 0.28426395939086296, + "grad_norm": 11.385386930454487, + "learning_rate": 6.300467418039222e-07, + "logits/chosen": -0.960631787776947, + "logits/rejected": -0.8502722382545471, + "logps/chosen": -396.5357360839844, + "logps/rejected": -541.553955078125, + "loss": 0.5093, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.8977466821670532, + "rewards/margins": 1.2623937129974365, + "rewards/rejected": -3.1601402759552, + "step": 133 + }, + { + "epoch": 0.28640128239380175, + "grad_norm": 11.957264036760781, + "learning_rate": 6.284685921086233e-07, + "logits/chosen": -0.8640112280845642, + "logits/rejected": -0.8963266611099243, + "logps/chosen": -384.329345703125, + "logps/rejected": -407.7789611816406, + "loss": 0.498, + "rewards/accuracies": 0.5625, + "rewards/chosen": -2.0386128425598145, + "rewards/margins": 0.3072957992553711, + "rewards/rejected": -2.3459086418151855, + "step": 134 + }, + { + "epoch": 0.2885386053967406, + "grad_norm": 11.759955525639521, + "learning_rate": 6.268748621297195e-07, + "logits/chosen": -0.8902631998062134, + "logits/rejected": -0.7775343656539917, + "logps/chosen": -403.541015625, + "logps/rejected": -534.9651489257812, + "loss": 0.4668, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.9903576374053955, + "rewards/margins": 1.1432865858078003, + "rewards/rejected": -3.1336441040039062, + "step": 135 + }, + { + "epoch": 0.29067592839967943, + "grad_norm": 11.950216444117117, + "learning_rate": 6.252656410362074e-07, + "logits/chosen": -0.9792582988739014, + "logits/rejected": -0.9614965915679932, + "logps/chosen": -561.1026611328125, + "logps/rejected": -663.7088012695312, + "loss": 0.4924, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.82108998298645, + "rewards/margins": 1.011682391166687, + "rewards/rejected": -3.8327722549438477, + "step": 136 + }, + { + "epoch": 0.2928132514026182, + "grad_norm": 10.772908788991131, + "learning_rate": 6.236410188638104e-07, + "logits/chosen": -1.053274393081665, + "logits/rejected": -0.9625363349914551, + "logps/chosen": -572.0177001953125, + "logps/rejected": -789.4164428710938, + "loss": 0.5162, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.667423725128174, + "rewards/margins": 2.069328546524048, + "rewards/rejected": -4.736752510070801, + "step": 137 + }, + { + "epoch": 0.29495057440555705, + "grad_norm": 9.953041872331532, + "learning_rate": 6.220010865099398e-07, + "logits/chosen": -1.0631476640701294, + "logits/rejected": -1.0828768014907837, + "logps/chosen": -395.8948669433594, + "logps/rejected": -478.61468505859375, + "loss": 0.4991, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.0671029090881348, + "rewards/margins": 0.7652881145477295, + "rewards/rejected": -2.8323915004730225, + "step": 138 + }, + { + "epoch": 0.29708789740849584, + "grad_norm": 9.56597037699992, + "learning_rate": 6.203459357286098e-07, + "logits/chosen": -1.0143028497695923, + "logits/rejected": -1.0154125690460205, + "logps/chosen": -517.6961669921875, + "logps/rejected": -596.7298583984375, + "loss": 0.4613, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.6147890090942383, + "rewards/margins": 0.7214634418487549, + "rewards/rejected": -3.336252212524414, + "step": 139 + }, + { + "epoch": 0.2992252204114347, + "grad_norm": 13.08709351718772, + "learning_rate": 6.186756591253038e-07, + "logits/chosen": -0.9290235042572021, + "logits/rejected": -0.9178426861763, + "logps/chosen": -527.4471435546875, + "logps/rejected": -568.3038330078125, + "loss": 0.5523, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.790428638458252, + "rewards/margins": 0.6073124408721924, + "rewards/rejected": -3.3977413177490234, + "step": 140 + }, + { + "epoch": 0.3013625434143735, + "grad_norm": 9.930492612600617, + "learning_rate": 6.169903501517932e-07, + "logits/chosen": -0.9746488928794861, + "logits/rejected": -0.9574599862098694, + "logps/chosen": -426.73760986328125, + "logps/rejected": -535.6727294921875, + "loss": 0.5198, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.180694580078125, + "rewards/margins": 0.977644145488739, + "rewards/rejected": -3.158338785171509, + "step": 141 + }, + { + "epoch": 0.3034998664173123, + "grad_norm": 14.86115549874432, + "learning_rate": 6.152901031009086e-07, + "logits/chosen": -0.9366909265518188, + "logits/rejected": -0.9158231019973755, + "logps/chosen": -489.0951232910156, + "logps/rejected": -574.0645141601562, + "loss": 0.5373, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.401134967803955, + "rewards/margins": 0.8239090442657471, + "rewards/rejected": -3.2250442504882812, + "step": 142 + }, + { + "epoch": 0.30563718942025114, + "grad_norm": 18.2841162585259, + "learning_rate": 6.135750131012639e-07, + "logits/chosen": -0.9252594709396362, + "logits/rejected": -0.8823320269584656, + "logps/chosen": -500.0283203125, + "logps/rejected": -582.243408203125, + "loss": 0.5163, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.57209849357605, + "rewards/margins": 0.7344849109649658, + "rewards/rejected": -3.3065831661224365, + "step": 143 + }, + { + "epoch": 0.3077745124231899, + "grad_norm": 10.361990112128243, + "learning_rate": 6.118451761119344e-07, + "logits/chosen": -0.9340752363204956, + "logits/rejected": -0.9739060997962952, + "logps/chosen": -686.3743286132812, + "logps/rejected": -827.3367919921875, + "loss": 0.4274, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.355973243713379, + "rewards/margins": 1.5688568353652954, + "rewards/rejected": -4.924829483032227, + "step": 144 + }, + { + "epoch": 0.30991183542612877, + "grad_norm": 14.742711073001722, + "learning_rate": 6.101006889170879e-07, + "logits/chosen": -0.9780830144882202, + "logits/rejected": -0.9924584031105042, + "logps/chosen": -611.8880615234375, + "logps/rejected": -783.9960327148438, + "loss": 0.4892, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.101553440093994, + "rewards/margins": 1.6389245986938477, + "rewards/rejected": -4.740478038787842, + "step": 145 + }, + { + "epoch": 0.3120491584290676, + "grad_norm": 11.760151720162954, + "learning_rate": 6.083416491205693e-07, + "logits/chosen": -0.9051653146743774, + "logits/rejected": -0.9137300252914429, + "logps/chosen": -518.42333984375, + "logps/rejected": -624.302490234375, + "loss": 0.4934, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.439613103866577, + "rewards/margins": 1.1922130584716797, + "rewards/rejected": -3.631826162338257, + "step": 146 + }, + { + "epoch": 0.3141864814320064, + "grad_norm": 12.767953140987824, + "learning_rate": 6.065681551404392e-07, + "logits/chosen": -0.9767911434173584, + "logits/rejected": -0.9007346034049988, + "logps/chosen": -498.14361572265625, + "logps/rejected": -551.3754272460938, + "loss": 0.5575, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.6738038063049316, + "rewards/margins": 0.7255844473838806, + "rewards/rejected": -3.399388313293457, + "step": 147 + }, + { + "epoch": 0.31632380443494523, + "grad_norm": 11.792817714413513, + "learning_rate": 6.047803062034688e-07, + "logits/chosen": -1.0714683532714844, + "logits/rejected": -1.0007489919662476, + "logps/chosen": -524.3717041015625, + "logps/rejected": -721.374267578125, + "loss": 0.5315, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.6513381004333496, + "rewards/margins": 1.8740673065185547, + "rewards/rejected": -4.525405406951904, + "step": 148 + }, + { + "epoch": 0.3184611274378841, + "grad_norm": 10.394135670198176, + "learning_rate": 6.029782023395871e-07, + "logits/chosen": -0.99624103307724, + "logits/rejected": -0.970421314239502, + "logps/chosen": -520.40771484375, + "logps/rejected": -551.2412109375, + "loss": 0.4518, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.831399917602539, + "rewards/margins": 0.5486774444580078, + "rewards/rejected": -3.380077362060547, + "step": 149 + }, + { + "epoch": 0.32059845044082286, + "grad_norm": 14.89234343098954, + "learning_rate": 6.011619443762841e-07, + "logits/chosen": -1.1248900890350342, + "logits/rejected": -1.1022447347640991, + "logps/chosen": -546.7406005859375, + "logps/rejected": -674.46484375, + "loss": 0.5543, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.7904787063598633, + "rewards/margins": 1.1329127550125122, + "rewards/rejected": -3.923391342163086, + "step": 150 + }, + { + "epoch": 0.3227357734437617, + "grad_norm": 10.760811605793574, + "learning_rate": 5.993316339329707e-07, + "logits/chosen": -0.8464213013648987, + "logits/rejected": -0.9315951466560364, + "logps/chosen": -557.2722778320312, + "logps/rejected": -562.7290649414062, + "loss": 0.5073, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.0413103103637695, + "rewards/margins": 0.24244722723960876, + "rewards/rejected": -3.2837576866149902, + "step": 151 + }, + { + "epoch": 0.3248730964467005, + "grad_norm": 12.468671998929876, + "learning_rate": 5.974873734152916e-07, + "logits/chosen": -0.9382638931274414, + "logits/rejected": -0.8903573751449585, + "logps/chosen": -517.4348754882812, + "logps/rejected": -586.1632690429688, + "loss": 0.559, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.7158446311950684, + "rewards/margins": 0.5773513317108154, + "rewards/rejected": -3.293196201324463, + "step": 152 + }, + { + "epoch": 0.3270104194496393, + "grad_norm": 10.71297529597072, + "learning_rate": 5.95629266009397e-07, + "logits/chosen": -0.9615129232406616, + "logits/rejected": -0.9363632202148438, + "logps/chosen": -530.434814453125, + "logps/rejected": -629.813720703125, + "loss": 0.4965, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.732804775238037, + "rewards/margins": 1.224104881286621, + "rewards/rejected": -3.9569101333618164, + "step": 153 + }, + { + "epoch": 0.32914774245257816, + "grad_norm": 12.327388201247391, + "learning_rate": 5.937574156761684e-07, + "logits/chosen": -0.9693203568458557, + "logits/rejected": -0.9414520263671875, + "logps/chosen": -467.2398681640625, + "logps/rejected": -549.6279296875, + "loss": 0.558, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.324397087097168, + "rewards/margins": 0.6942296624183655, + "rewards/rejected": -3.018627166748047, + "step": 154 + }, + { + "epoch": 0.33128506545551695, + "grad_norm": 11.452079247996043, + "learning_rate": 5.918719271454026e-07, + "logits/chosen": -0.9070168137550354, + "logits/rejected": -0.9717684984207153, + "logps/chosen": -463.07073974609375, + "logps/rejected": -531.9006958007812, + "loss": 0.4757, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.246826171875, + "rewards/margins": 0.8463835120201111, + "rewards/rejected": -3.0932095050811768, + "step": 155 + }, + { + "epoch": 0.3334223884584558, + "grad_norm": 10.387010221236869, + "learning_rate": 5.89972905909952e-07, + "logits/chosen": -0.9737594127655029, + "logits/rejected": -0.9816542267799377, + "logps/chosen": -482.228515625, + "logps/rejected": -550.7467041015625, + "loss": 0.4944, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.8164095878601074, + "rewards/margins": 0.6258185505867004, + "rewards/rejected": -3.442227840423584, + "step": 156 + }, + { + "epoch": 0.3355597114613946, + "grad_norm": 13.81426757522365, + "learning_rate": 5.880604582198217e-07, + "logits/chosen": -0.8456093072891235, + "logits/rejected": -0.831813395023346, + "logps/chosen": -598.4921875, + "logps/rejected": -722.5546875, + "loss": 0.4857, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.0666327476501465, + "rewards/margins": 1.3944388628005981, + "rewards/rejected": -4.461071014404297, + "step": 157 + }, + { + "epoch": 0.3376970344643334, + "grad_norm": 12.739583339970373, + "learning_rate": 5.86134691076226e-07, + "logits/chosen": -0.9977485537528992, + "logits/rejected": -0.9227933287620544, + "logps/chosen": -458.33758544921875, + "logps/rejected": -590.2880249023438, + "loss": 0.4951, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.321625232696533, + "rewards/margins": 1.1777639389038086, + "rewards/rejected": -3.499389410018921, + "step": 158 + }, + { + "epoch": 0.33983435746727225, + "grad_norm": 11.40592684455961, + "learning_rate": 5.841957122256004e-07, + "logits/chosen": -0.9870405793190002, + "logits/rejected": -0.9605780839920044, + "logps/chosen": -460.610107421875, + "logps/rejected": -543.5918579101562, + "loss": 0.4798, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.591230869293213, + "rewards/margins": 0.8468169569969177, + "rewards/rejected": -3.4380481243133545, + "step": 159 + }, + { + "epoch": 0.34197168047021104, + "grad_norm": 11.54298974939068, + "learning_rate": 5.822436301535739e-07, + "logits/chosen": -0.9439342617988586, + "logits/rejected": -0.896612823009491, + "logps/chosen": -549.1138916015625, + "logps/rejected": -611.7896728515625, + "loss": 0.4867, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.6459498405456543, + "rewards/margins": 0.6619236469268799, + "rewards/rejected": -3.307873487472534, + "step": 160 + }, + { + "epoch": 0.3441090034731499, + "grad_norm": 10.028108962774086, + "learning_rate": 5.802785540788994e-07, + "logits/chosen": -1.018284797668457, + "logits/rejected": -0.9662111401557922, + "logps/chosen": -477.1954650878906, + "logps/rejected": -642.691650390625, + "loss": 0.4616, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.5217745304107666, + "rewards/margins": 1.391313910484314, + "rewards/rejected": -3.913088321685791, + "step": 161 + }, + { + "epoch": 0.3462463264760887, + "grad_norm": 10.240941578340253, + "learning_rate": 5.783005939473424e-07, + "logits/chosen": -0.8555217981338501, + "logits/rejected": -0.8488653898239136, + "logps/chosen": -490.9669189453125, + "logps/rejected": -589.8624267578125, + "loss": 0.474, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.722303628921509, + "rewards/margins": 0.9005026817321777, + "rewards/rejected": -3.6228065490722656, + "step": 162 + }, + { + "epoch": 0.3483836494790275, + "grad_norm": 11.409205514657348, + "learning_rate": 5.763098604255298e-07, + "logits/chosen": -0.8463901281356812, + "logits/rejected": -0.8225279450416565, + "logps/chosen": -470.57440185546875, + "logps/rejected": -639.9716796875, + "loss": 0.4536, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.5086305141448975, + "rewards/margins": 1.750978946685791, + "rewards/rejected": -4.259609222412109, + "step": 163 + }, + { + "epoch": 0.35052097248196634, + "grad_norm": 10.927823097709238, + "learning_rate": 5.743064648947584e-07, + "logits/chosen": -0.7492021322250366, + "logits/rejected": -0.7506992816925049, + "logps/chosen": -561.3836669921875, + "logps/rejected": -710.4789428710938, + "loss": 0.5088, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.9847238063812256, + "rewards/margins": 1.428779125213623, + "rewards/rejected": -4.413503170013428, + "step": 164 + }, + { + "epoch": 0.3526582954849052, + "grad_norm": 13.549577455647025, + "learning_rate": 5.722905194447625e-07, + "logits/chosen": -1.0247427225112915, + "logits/rejected": -1.0630146265029907, + "logps/chosen": -696.59228515625, + "logps/rejected": -761.5941772460938, + "loss": 0.5028, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.713867425918579, + "rewards/margins": 0.7951303124427795, + "rewards/rejected": -4.508997440338135, + "step": 165 + }, + { + "epoch": 0.35479561848784397, + "grad_norm": 15.326243524128424, + "learning_rate": 5.702621368674431e-07, + "logits/chosen": -0.9843580722808838, + "logits/rejected": -0.9505161643028259, + "logps/chosen": -439.31622314453125, + "logps/rejected": -539.7349243164062, + "loss": 0.5054, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.5671231746673584, + "rewards/margins": 0.8157074451446533, + "rewards/rejected": -3.3828306198120117, + "step": 166 + }, + { + "epoch": 0.3569329414907828, + "grad_norm": 13.81282026576703, + "learning_rate": 5.682214306505567e-07, + "logits/chosen": -0.7752354145050049, + "logits/rejected": -0.831868052482605, + "logps/chosen": -618.5186157226562, + "logps/rejected": -734.19580078125, + "loss": 0.4584, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.667295217514038, + "rewards/margins": 1.2412769794464111, + "rewards/rejected": -4.908572196960449, + "step": 167 + }, + { + "epoch": 0.3590702644937216, + "grad_norm": 12.331430233086655, + "learning_rate": 5.661685149713663e-07, + "logits/chosen": -0.8932394981384277, + "logits/rejected": -0.8569494485855103, + "logps/chosen": -504.1289367675781, + "logps/rejected": -628.0838012695312, + "loss": 0.5174, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.809536933898926, + "rewards/margins": 1.0781981945037842, + "rewards/rejected": -3.88773512840271, + "step": 168 + }, + { + "epoch": 0.36120758749666043, + "grad_norm": 13.419219251759696, + "learning_rate": 5.641035046902522e-07, + "logits/chosen": -0.8772855997085571, + "logits/rejected": -0.8467257618904114, + "logps/chosen": -470.7606201171875, + "logps/rejected": -565.2396240234375, + "loss": 0.4684, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.8519928455352783, + "rewards/margins": 0.8433955907821655, + "rewards/rejected": -3.695388078689575, + "step": 169 + }, + { + "epoch": 0.36334491049959927, + "grad_norm": 12.695684401858434, + "learning_rate": 5.620265153442869e-07, + "logits/chosen": -1.0130642652511597, + "logits/rejected": -0.9847390055656433, + "logps/chosen": -514.5957641601562, + "logps/rejected": -585.168212890625, + "loss": 0.469, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.0407803058624268, + "rewards/margins": 0.6757410168647766, + "rewards/rejected": -3.7165212631225586, + "step": 170 + }, + { + "epoch": 0.36548223350253806, + "grad_norm": 12.147780619595629, + "learning_rate": 5.599376631407699e-07, + "logits/chosen": -0.9069080948829651, + "logits/rejected": -0.8009949922561646, + "logps/chosen": -547.1576538085938, + "logps/rejected": -679.2531127929688, + "loss": 0.4299, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.197631359100342, + "rewards/margins": 1.0914721488952637, + "rewards/rejected": -4.2891035079956055, + "step": 171 + }, + { + "epoch": 0.3676195565054769, + "grad_norm": 18.51274657202258, + "learning_rate": 5.578370649507255e-07, + "logits/chosen": -0.6547280550003052, + "logits/rejected": -0.6987937092781067, + "logps/chosen": -577.7064819335938, + "logps/rejected": -660.9352416992188, + "loss": 0.5682, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.2265751361846924, + "rewards/margins": 0.9371775388717651, + "rewards/rejected": -4.163752555847168, + "step": 172 + }, + { + "epoch": 0.36975687950841574, + "grad_norm": 13.62299944344872, + "learning_rate": 5.557248383023656e-07, + "logits/chosen": -0.9801725745201111, + "logits/rejected": -0.945483386516571, + "logps/chosen": -449.10614013671875, + "logps/rejected": -567.4400634765625, + "loss": 0.4765, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.63840389251709, + "rewards/margins": 1.1649121046066284, + "rewards/rejected": -3.803316116333008, + "step": 173 + }, + { + "epoch": 0.3718942025113545, + "grad_norm": 12.16862925856691, + "learning_rate": 5.536011013745121e-07, + "logits/chosen": -0.9143638610839844, + "logits/rejected": -0.9123052358627319, + "logps/chosen": -483.41357421875, + "logps/rejected": -624.1827392578125, + "loss": 0.459, + "rewards/accuracies": 0.875, + "rewards/chosen": -2.4780359268188477, + "rewards/margins": 1.3084805011749268, + "rewards/rejected": -3.7865164279937744, + "step": 174 + }, + { + "epoch": 0.37403152551429336, + "grad_norm": 11.507324455166394, + "learning_rate": 5.514659729899857e-07, + "logits/chosen": -0.7952810525894165, + "logits/rejected": -0.7528566718101501, + "logps/chosen": -498.6630554199219, + "logps/rejected": -630.5216064453125, + "loss": 0.4045, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.8591041564941406, + "rewards/margins": 1.1144087314605713, + "rewards/rejected": -3.973512649536133, + "step": 175 + }, + { + "epoch": 0.37616884851723215, + "grad_norm": 12.170272129622308, + "learning_rate": 5.493195726089582e-07, + "logits/chosen": -0.9865118265151978, + "logits/rejected": -0.9716078639030457, + "logps/chosen": -543.14501953125, + "logps/rejected": -683.6224365234375, + "loss": 0.4348, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.054544448852539, + "rewards/margins": 1.5414537191390991, + "rewards/rejected": -4.595998287200928, + "step": 176 + }, + { + "epoch": 0.378306171520171, + "grad_norm": 13.22726269166873, + "learning_rate": 5.471620203222677e-07, + "logits/chosen": -0.9932882189750671, + "logits/rejected": -0.991521418094635, + "logps/chosen": -507.20574951171875, + "logps/rejected": -601.6077880859375, + "loss": 0.4227, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.048675060272217, + "rewards/margins": 0.8585546016693115, + "rewards/rejected": -3.9072299003601074, + "step": 177 + }, + { + "epoch": 0.3804434945231098, + "grad_norm": 11.607213817333788, + "learning_rate": 5.449934368447002e-07, + "logits/chosen": -0.8857173919677734, + "logits/rejected": -0.9733960628509521, + "logps/chosen": -627.8464965820312, + "logps/rejected": -713.438720703125, + "loss": 0.4544, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.520371437072754, + "rewards/margins": 1.112282156944275, + "rewards/rejected": -4.63265323638916, + "step": 178 + }, + { + "epoch": 0.3825808175260486, + "grad_norm": 12.188016147935972, + "learning_rate": 5.428139435082358e-07, + "logits/chosen": -0.7584793567657471, + "logits/rejected": -0.8010637760162354, + "logps/chosen": -566.6948852539062, + "logps/rejected": -713.6017456054688, + "loss": 0.4106, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.4136064052581787, + "rewards/margins": 1.416290044784546, + "rewards/rejected": -4.829895973205566, + "step": 179 + }, + { + "epoch": 0.38471814052898745, + "grad_norm": 13.803380819286584, + "learning_rate": 5.406236622552594e-07, + "logits/chosen": -0.8117521405220032, + "logits/rejected": -0.8442290425300598, + "logps/chosen": -658.14990234375, + "logps/rejected": -791.3058471679688, + "loss": 0.4795, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.9962399005889893, + "rewards/margins": 1.3741449117660522, + "rewards/rejected": -5.370384693145752, + "step": 180 + }, + { + "epoch": 0.38685546353192624, + "grad_norm": 19.74453585010364, + "learning_rate": 5.384227156317388e-07, + "logits/chosen": -0.9625721573829651, + "logits/rejected": -1.0246632099151611, + "logps/chosen": -570.2412109375, + "logps/rejected": -627.0347900390625, + "loss": 0.5302, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.3639683723449707, + "rewards/margins": 0.4717191457748413, + "rewards/rejected": -3.8356876373291016, + "step": 181 + }, + { + "epoch": 0.3889927865348651, + "grad_norm": 12.180825522167304, + "learning_rate": 5.362112267803678e-07, + "logits/chosen": -1.1118032932281494, + "logits/rejected": -1.0548309087753296, + "logps/chosen": -561.20654296875, + "logps/rejected": -801.3348388671875, + "loss": 0.407, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.0669779777526855, + "rewards/margins": 2.297924041748047, + "rewards/rejected": -5.364901542663574, + "step": 182 + }, + { + "epoch": 0.3911301095378039, + "grad_norm": 15.493196005683158, + "learning_rate": 5.339893194336765e-07, + "logits/chosen": -0.9748885035514832, + "logits/rejected": -0.9212169647216797, + "logps/chosen": -556.37451171875, + "logps/rejected": -747.4850463867188, + "loss": 0.434, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.1176629066467285, + "rewards/margins": 2.0257248878479004, + "rewards/rejected": -5.143387794494629, + "step": 183 + }, + { + "epoch": 0.3932674325407427, + "grad_norm": 12.834279479584271, + "learning_rate": 5.317571179071092e-07, + "logits/chosen": -1.0676939487457275, + "logits/rejected": -1.0940359830856323, + "logps/chosen": -547.4573364257812, + "logps/rejected": -625.5507202148438, + "loss": 0.4441, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.1368393898010254, + "rewards/margins": 0.7552158236503601, + "rewards/rejected": -3.8920555114746094, + "step": 184 + }, + { + "epoch": 0.39540475554368154, + "grad_norm": 15.858497539317051, + "learning_rate": 5.295147470920672e-07, + "logits/chosen": -0.8825098276138306, + "logits/rejected": -0.8697827458381653, + "logps/chosen": -606.1354370117188, + "logps/rejected": -752.2478637695312, + "loss": 0.4948, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.499366521835327, + "rewards/margins": 1.4578311443328857, + "rewards/rejected": -4.957198143005371, + "step": 185 + }, + { + "epoch": 0.3975420785466204, + "grad_norm": 15.81390161765314, + "learning_rate": 5.27262332448923e-07, + "logits/chosen": -0.9162149429321289, + "logits/rejected": -0.8807965517044067, + "logps/chosen": -588.79736328125, + "logps/rejected": -775.908447265625, + "loss": 0.4836, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.620396852493286, + "rewards/margins": 1.820716142654419, + "rewards/rejected": -5.441113471984863, + "step": 186 + }, + { + "epoch": 0.39967940154955917, + "grad_norm": 11.930553509668082, + "learning_rate": 5.25e-07, + "logits/chosen": -0.878287136554718, + "logits/rejected": -0.8447045683860779, + "logps/chosen": -588.80908203125, + "logps/rejected": -710.580078125, + "loss": 0.4217, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.770730495452881, + "rewards/margins": 1.0582211017608643, + "rewards/rejected": -4.828950881958008, + "step": 187 + }, + { + "epoch": 0.401816724552498, + "grad_norm": 19.78691767583468, + "learning_rate": 5.227278763225213e-07, + "logits/chosen": -1.002894639968872, + "logits/rejected": -0.9999670386314392, + "logps/chosen": -729.7056884765625, + "logps/rejected": -895.4324340820312, + "loss": 0.488, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.301171779632568, + "rewards/margins": 1.7632997035980225, + "rewards/rejected": -6.06447172164917, + "step": 188 + }, + { + "epoch": 0.4039540475554368, + "grad_norm": 15.17666246755368, + "learning_rate": 5.204460885415285e-07, + "logits/chosen": -0.9192298650741577, + "logits/rejected": -0.9201673269271851, + "logps/chosen": -612.3890380859375, + "logps/rejected": -722.1416625976562, + "loss": 0.4896, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.5543270111083984, + "rewards/margins": 1.225655198097229, + "rewards/rejected": -4.779982089996338, + "step": 189 + }, + { + "epoch": 0.40609137055837563, + "grad_norm": 12.423293515766808, + "learning_rate": 5.181547643227682e-07, + "logits/chosen": -0.8379946947097778, + "logits/rejected": -0.8106329441070557, + "logps/chosen": -546.194580078125, + "logps/rejected": -636.456298828125, + "loss": 0.4722, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.497601270675659, + "rewards/margins": 0.9037814736366272, + "rewards/rejected": -4.4013824462890625, + "step": 190 + }, + { + "epoch": 0.40822869356131447, + "grad_norm": 16.79590350569075, + "learning_rate": 5.158540318655495e-07, + "logits/chosen": -1.0028809309005737, + "logits/rejected": -0.9536342024803162, + "logps/chosen": -669.4102783203125, + "logps/rejected": -854.4193115234375, + "loss": 0.4629, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.044215679168701, + "rewards/margins": 1.6450707912445068, + "rewards/rejected": -5.689286708831787, + "step": 191 + }, + { + "epoch": 0.41036601656425326, + "grad_norm": 14.164634270897476, + "learning_rate": 5.135440198955717e-07, + "logits/chosen": -0.97246915102005, + "logits/rejected": -0.9198447465896606, + "logps/chosen": -513.80859375, + "logps/rejected": -543.1536865234375, + "loss": 0.4983, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.1685631275177, + "rewards/margins": 0.3603837192058563, + "rewards/rejected": -3.528946876525879, + "step": 192 + }, + { + "epoch": 0.4125033395671921, + "grad_norm": 15.329408000607371, + "learning_rate": 5.112248576577213e-07, + "logits/chosen": -0.7749034762382507, + "logits/rejected": -0.8009266257286072, + "logps/chosen": -596.5591430664062, + "logps/rejected": -797.4959716796875, + "loss": 0.4978, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.661137342453003, + "rewards/margins": 1.7927237749099731, + "rewards/rejected": -5.453861236572266, + "step": 193 + }, + { + "epoch": 0.41464066257013094, + "grad_norm": 14.042005201292891, + "learning_rate": 5.088966749088414e-07, + "logits/chosen": -1.116222858428955, + "logits/rejected": -1.090318202972412, + "logps/chosen": -603.7730102539062, + "logps/rejected": -657.4857177734375, + "loss": 0.4895, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.191464424133301, + "rewards/margins": 0.7639944553375244, + "rewards/rejected": -3.9554591178894043, + "step": 194 + }, + { + "epoch": 0.4167779855730697, + "grad_norm": 13.675394100746447, + "learning_rate": 5.065596019104713e-07, + "logits/chosen": -1.0994431972503662, + "logits/rejected": -1.091101884841919, + "logps/chosen": -700.6111450195312, + "logps/rejected": -847.7877197265625, + "loss": 0.4107, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.003596305847168, + "rewards/margins": 1.4772356748580933, + "rewards/rejected": -5.480831623077393, + "step": 195 + }, + { + "epoch": 0.41891530857600856, + "grad_norm": 18.55076809192975, + "learning_rate": 5.042137694215592e-07, + "logits/chosen": -0.9493328928947449, + "logits/rejected": -1.022355079650879, + "logps/chosen": -641.4091796875, + "logps/rejected": -696.9602661132812, + "loss": 0.618, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.780533790588379, + "rewards/margins": 0.6779485940933228, + "rewards/rejected": -4.458482265472412, + "step": 196 + }, + { + "epoch": 0.42105263157894735, + "grad_norm": 12.163973418481676, + "learning_rate": 5.018593086911453e-07, + "logits/chosen": -1.0403311252593994, + "logits/rejected": -1.069019079208374, + "logps/chosen": -671.8824462890625, + "logps/rejected": -849.7545166015625, + "loss": 0.4197, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.7160799503326416, + "rewards/margins": 1.7244200706481934, + "rewards/rejected": -5.440499782562256, + "step": 197 + }, + { + "epoch": 0.4231899545818862, + "grad_norm": 13.736070000914967, + "learning_rate": 4.994963514510188e-07, + "logits/chosen": -1.0466636419296265, + "logits/rejected": -0.9742124676704407, + "logps/chosen": -536.3780517578125, + "logps/rejected": -622.6585083007812, + "loss": 0.4457, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.165748119354248, + "rewards/margins": 0.7600272297859192, + "rewards/rejected": -3.9257755279541016, + "step": 198 + }, + { + "epoch": 0.425327277584825, + "grad_norm": 11.871725745717265, + "learning_rate": 4.971250299083479e-07, + "logits/chosen": -0.9257810711860657, + "logits/rejected": -0.9252554774284363, + "logps/chosen": -556.8322143554688, + "logps/rejected": -656.6981201171875, + "loss": 0.4155, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.365008592605591, + "rewards/margins": 1.075837254524231, + "rewards/rejected": -4.440845966339111, + "step": 199 + }, + { + "epoch": 0.4274646005877638, + "grad_norm": 12.50996465065088, + "learning_rate": 4.947454767382822e-07, + "logits/chosen": -0.9889892339706421, + "logits/rejected": -1.004003882408142, + "logps/chosen": -499.806884765625, + "logps/rejected": -617.0054321289062, + "loss": 0.5021, + "rewards/accuracies": 0.625, + "rewards/chosen": -2.9506893157958984, + "rewards/margins": 1.0361154079437256, + "rewards/rejected": -3.986804485321045, + "step": 200 + }, + { + "epoch": 0.4274646005877638, + "eval_logits/chosen": -1.084425926208496, + "eval_logits/rejected": -1.103520393371582, + "eval_logps/chosen": -613.6640625, + "eval_logps/rejected": -715.6145629882812, + "eval_loss": 0.44827935099601746, + "eval_rewards/accuracies": 0.8024193644523621, + "eval_rewards/chosen": -3.4053404331207275, + "eval_rewards/margins": 1.080467700958252, + "eval_rewards/rejected": -4.4858078956604, + "eval_runtime": 126.9937, + "eval_samples_per_second": 15.442, + "eval_steps_per_second": 0.488, + "step": 200 + }, + { + "epoch": 0.42960192359070265, + "grad_norm": 13.246746821586836, + "learning_rate": 4.923578250765301e-07, + "logits/chosen": -1.0361019372940063, + "logits/rejected": -1.0069564580917358, + "logps/chosen": -630.4815063476562, + "logps/rejected": -819.7576293945312, + "loss": 0.4381, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.616086483001709, + "rewards/margins": 1.9921762943267822, + "rewards/rejected": -5.608262538909912, + "step": 201 + }, + { + "epoch": 0.4317392465936415, + "grad_norm": 13.986462198608523, + "learning_rate": 4.899622085119092e-07, + "logits/chosen": -1.0016858577728271, + "logits/rejected": -1.0090727806091309, + "logps/chosen": -639.8945922851562, + "logps/rejected": -797.4194946289062, + "loss": 0.389, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.3241751194000244, + "rewards/margins": 1.6805753707885742, + "rewards/rejected": -5.0047502517700195, + "step": 202 + }, + { + "epoch": 0.4338765695965803, + "grad_norm": 12.938400299110988, + "learning_rate": 4.875587610788733e-07, + "logits/chosen": -1.0168877840042114, + "logits/rejected": -1.0365827083587646, + "logps/chosen": -593.1552124023438, + "logps/rejected": -752.401611328125, + "loss": 0.4125, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.574855327606201, + "rewards/margins": 1.3029221296310425, + "rewards/rejected": -4.877778053283691, + "step": 203 + }, + { + "epoch": 0.4360138925995191, + "grad_norm": 12.350696519100337, + "learning_rate": 4.851476172500112e-07, + "logits/chosen": -1.0002954006195068, + "logits/rejected": -0.9592558145523071, + "logps/chosen": -691.1764526367188, + "logps/rejected": -894.4859619140625, + "loss": 0.4075, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.358014106750488, + "rewards/margins": 1.7022799253463745, + "rewards/rejected": -6.060294151306152, + "step": 204 + }, + { + "epoch": 0.4381512156024579, + "grad_norm": 14.780877866432451, + "learning_rate": 4.827289119285249e-07, + "logits/chosen": -0.9795655608177185, + "logits/rejected": -0.9753063917160034, + "logps/chosen": -560.5239868164062, + "logps/rejected": -701.3786010742188, + "loss": 0.466, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.2604007720947266, + "rewards/margins": 1.3381632566452026, + "rewards/rejected": -4.5985636711120605, + "step": 205 + }, + { + "epoch": 0.44028853860539674, + "grad_norm": 22.9673286565427, + "learning_rate": 4.803027804406808e-07, + "logits/chosen": -0.7228925228118896, + "logits/rejected": -0.8121004700660706, + "logps/chosen": -680.5167236328125, + "logps/rejected": -703.938232421875, + "loss": 0.559, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.237366199493408, + "rewards/margins": 0.45848214626312256, + "rewards/rejected": -4.695847988128662, + "step": 206 + }, + { + "epoch": 0.4424258616083356, + "grad_norm": 18.405605350322162, + "learning_rate": 4.778693585282384e-07, + "logits/chosen": -1.0253394842147827, + "logits/rejected": -1.0438570976257324, + "logps/chosen": -631.9534912109375, + "logps/rejected": -705.13623046875, + "loss": 0.5002, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.731057643890381, + "rewards/margins": 0.8837456107139587, + "rewards/rejected": -4.614803314208984, + "step": 207 + }, + { + "epoch": 0.44456318461127436, + "grad_norm": 15.70410271681006, + "learning_rate": 4.75428782340855e-07, + "logits/chosen": -1.0454537868499756, + "logits/rejected": -1.1345258951187134, + "logps/chosen": -641.6697998046875, + "logps/rejected": -731.5447387695312, + "loss": 0.4413, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.9197850227355957, + "rewards/margins": 0.858109712600708, + "rewards/rejected": -4.777895450592041, + "step": 208 + }, + { + "epoch": 0.4467005076142132, + "grad_norm": 13.037823833248444, + "learning_rate": 4.7298118842846993e-07, + "logits/chosen": -0.9263153076171875, + "logits/rejected": -0.9316372871398926, + "logps/chosen": -522.490966796875, + "logps/rejected": -635.503173828125, + "loss": 0.3848, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.2573442459106445, + "rewards/margins": 0.9754186272621155, + "rewards/rejected": -4.232762336730957, + "step": 209 + }, + { + "epoch": 0.448837830617152, + "grad_norm": 15.711193208519104, + "learning_rate": 4.705267137336622e-07, + "logits/chosen": -1.0621023178100586, + "logits/rejected": -1.0041028261184692, + "logps/chosen": -799.73974609375, + "logps/rejected": -927.7545776367188, + "loss": 0.4351, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.812376022338867, + "rewards/margins": 1.1584749221801758, + "rewards/rejected": -5.970850944519043, + "step": 210 + }, + { + "epoch": 0.45097515362009083, + "grad_norm": 13.021464052671469, + "learning_rate": 4.680654955839905e-07, + "logits/chosen": -1.0352355241775513, + "logits/rejected": -1.0864734649658203, + "logps/chosen": -679.844970703125, + "logps/rejected": -842.2390747070312, + "loss": 0.3926, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.970327138900757, + "rewards/margins": 1.7980788946151733, + "rewards/rejected": -5.768405914306641, + "step": 211 + }, + { + "epoch": 0.45311247662302967, + "grad_norm": 13.400793392671966, + "learning_rate": 4.655976716843085e-07, + "logits/chosen": -0.9447561502456665, + "logits/rejected": -0.9419984221458435, + "logps/chosen": -577.1600341796875, + "logps/rejected": -626.2113647460938, + "loss": 0.4097, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.421278476715088, + "rewards/margins": 0.7281789183616638, + "rewards/rejected": -4.1494574546813965, + "step": 212 + }, + { + "epoch": 0.45524979962596845, + "grad_norm": 15.76259483264769, + "learning_rate": 4.63123380109061e-07, + "logits/chosen": -0.9420887231826782, + "logits/rejected": -0.9258626699447632, + "logps/chosen": -604.66064453125, + "logps/rejected": -688.6736450195312, + "loss": 0.5374, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.727839708328247, + "rewards/margins": 0.6107276082038879, + "rewards/rejected": -4.33856725692749, + "step": 213 + }, + { + "epoch": 0.4573871226289073, + "grad_norm": 14.47796429675634, + "learning_rate": 4.6064275929455854e-07, + "logits/chosen": -1.012191653251648, + "logits/rejected": -0.9834191799163818, + "logps/chosen": -539.5471801757812, + "logps/rejected": -643.7244873046875, + "loss": 0.3953, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.355208396911621, + "rewards/margins": 0.9154967665672302, + "rewards/rejected": -4.270705223083496, + "step": 214 + }, + { + "epoch": 0.45952444563184613, + "grad_norm": 18.127660409487774, + "learning_rate": 4.581559480312316e-07, + "logits/chosen": -0.9188220500946045, + "logits/rejected": -0.8601971864700317, + "logps/chosen": -562.9246826171875, + "logps/rejected": -646.825439453125, + "loss": 0.5093, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.4996066093444824, + "rewards/margins": 0.7593184113502502, + "rewards/rejected": -4.258925437927246, + "step": 215 + }, + { + "epoch": 0.4616617686347849, + "grad_norm": 14.332440562045605, + "learning_rate": 4.556630854558657e-07, + "logits/chosen": -1.0821930170059204, + "logits/rejected": -1.0690282583236694, + "logps/chosen": -612.6143188476562, + "logps/rejected": -785.2466430664062, + "loss": 0.4259, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.40315580368042, + "rewards/margins": 1.751339316368103, + "rewards/rejected": -5.1544952392578125, + "step": 216 + }, + { + "epoch": 0.46379909163772376, + "grad_norm": 13.6646330391651, + "learning_rate": 4.5316431104381647e-07, + "logits/chosen": -1.0587506294250488, + "logits/rejected": -1.0372809171676636, + "logps/chosen": -491.8074951171875, + "logps/rejected": -638.18212890625, + "loss": 0.4567, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.595409870147705, + "rewards/margins": 1.3147836923599243, + "rewards/rejected": -3.910193681716919, + "step": 217 + }, + { + "epoch": 0.46593641464066254, + "grad_norm": 16.27324594458174, + "learning_rate": 4.50659764601206e-07, + "logits/chosen": -1.1704292297363281, + "logits/rejected": -1.1407532691955566, + "logps/chosen": -635.9962158203125, + "logps/rejected": -709.2085571289062, + "loss": 0.5169, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.4440693855285645, + "rewards/margins": 0.8417525887489319, + "rewards/rejected": -4.285822868347168, + "step": 218 + }, + { + "epoch": 0.4680737376436014, + "grad_norm": 14.30405716244965, + "learning_rate": 4.4814958625710095e-07, + "logits/chosen": -0.9373888969421387, + "logits/rejected": -0.9414705634117126, + "logps/chosen": -641.3363037109375, + "logps/rejected": -782.9466552734375, + "loss": 0.4214, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.41910719871521, + "rewards/margins": 1.5010746717453003, + "rewards/rejected": -4.920182228088379, + "step": 219 + }, + { + "epoch": 0.4702110606465402, + "grad_norm": 12.888240768536523, + "learning_rate": 4.4563391645567203e-07, + "logits/chosen": -0.9721742272377014, + "logits/rejected": -0.9880850315093994, + "logps/chosen": -563.5908813476562, + "logps/rejected": -648.4769897460938, + "loss": 0.4525, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.4122231006622314, + "rewards/margins": 0.8691967725753784, + "rewards/rejected": -4.28141975402832, + "step": 220 + }, + { + "epoch": 0.472348383649479, + "grad_norm": 13.808905917996086, + "learning_rate": 4.4311289594833626e-07, + "logits/chosen": -1.062425136566162, + "logits/rejected": -1.0666816234588623, + "logps/chosen": -425.4259338378906, + "logps/rejected": -508.6617736816406, + "loss": 0.3815, + "rewards/accuracies": 0.6875, + "rewards/chosen": -2.534140110015869, + "rewards/margins": 0.9357590079307556, + "rewards/rejected": -3.4698996543884277, + "step": 221 + }, + { + "epoch": 0.47448570665241785, + "grad_norm": 14.257159662099976, + "learning_rate": 4.405866657858822e-07, + "logits/chosen": -1.0133253335952759, + "logits/rejected": -1.0463453531265259, + "logps/chosen": -577.285888671875, + "logps/rejected": -644.2965087890625, + "loss": 0.4452, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.20715594291687, + "rewards/margins": 0.7728344202041626, + "rewards/rejected": -3.979990243911743, + "step": 222 + }, + { + "epoch": 0.4766230296553567, + "grad_norm": 12.854510596268446, + "learning_rate": 4.38055367310578e-07, + "logits/chosen": -0.985044538974762, + "logits/rejected": -0.9488839507102966, + "logps/chosen": -645.5025024414062, + "logps/rejected": -869.3623046875, + "loss": 0.3977, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.6057004928588867, + "rewards/margins": 1.8732173442840576, + "rewards/rejected": -5.478918075561523, + "step": 223 + }, + { + "epoch": 0.4787603526582955, + "grad_norm": 14.869442818796234, + "learning_rate": 4.355191421482629e-07, + "logits/chosen": -1.1076860427856445, + "logits/rejected": -1.0764734745025635, + "logps/chosen": -516.0100708007812, + "logps/rejected": -644.3858642578125, + "loss": 0.4573, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.1032497882843018, + "rewards/margins": 1.1473891735076904, + "rewards/rejected": -4.250638961791992, + "step": 224 + }, + { + "epoch": 0.4808976756612343, + "grad_norm": 16.43448798480314, + "learning_rate": 4.329781322004241e-07, + "logits/chosen": -1.0606333017349243, + "logits/rejected": -1.0501140356063843, + "logps/chosen": -663.1077880859375, + "logps/rejected": -874.1226806640625, + "loss": 0.4973, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.6670713424682617, + "rewards/margins": 1.8036597967147827, + "rewards/rejected": -5.470730781555176, + "step": 225 + }, + { + "epoch": 0.4830349986641731, + "grad_norm": 15.477278289904351, + "learning_rate": 4.304324796362568e-07, + "logits/chosen": -1.026381015777588, + "logits/rejected": -0.9911444187164307, + "logps/chosen": -616.499267578125, + "logps/rejected": -757.9083862304688, + "loss": 0.4483, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.533419609069824, + "rewards/margins": 1.2801600694656372, + "rewards/rejected": -4.81358003616333, + "step": 226 + }, + { + "epoch": 0.48517232166711194, + "grad_norm": 14.181376472254154, + "learning_rate": 4.2788232688471e-07, + "logits/chosen": -1.0257827043533325, + "logits/rejected": -1.030511498451233, + "logps/chosen": -623.668212890625, + "logps/rejected": -722.494873046875, + "loss": 0.4315, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.7645962238311768, + "rewards/margins": 1.0766507387161255, + "rewards/rejected": -4.841247081756592, + "step": 227 + }, + { + "epoch": 0.4873096446700508, + "grad_norm": 18.128852851106277, + "learning_rate": 4.253278166265177e-07, + "logits/chosen": -1.0358821153640747, + "logits/rejected": -1.0768516063690186, + "logps/chosen": -567.903564453125, + "logps/rejected": -689.8753051757812, + "loss": 0.4725, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.21205997467041, + "rewards/margins": 1.3463387489318848, + "rewards/rejected": -4.558398723602295, + "step": 228 + }, + { + "epoch": 0.48944696767298956, + "grad_norm": 13.499906597454904, + "learning_rate": 4.2276909178621567e-07, + "logits/chosen": -0.9607468843460083, + "logits/rejected": -0.9298718571662903, + "logps/chosen": -749.897705078125, + "logps/rejected": -971.5405883789062, + "loss": 0.3638, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.647284030914307, + "rewards/margins": 2.187747001647949, + "rewards/rejected": -6.835031032562256, + "step": 229 + }, + { + "epoch": 0.4915842906759284, + "grad_norm": 15.151757518766736, + "learning_rate": 4.2020629552414545e-07, + "logits/chosen": -1.0237997770309448, + "logits/rejected": -0.9573734998703003, + "logps/chosen": -746.3142700195312, + "logps/rejected": -845.5519409179688, + "loss": 0.4511, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.615874767303467, + "rewards/margins": 1.053206205368042, + "rewards/rejected": -5.669081211090088, + "step": 230 + }, + { + "epoch": 0.49372161367886724, + "grad_norm": 15.54384712409316, + "learning_rate": 4.176395712284436e-07, + "logits/chosen": -1.0997838973999023, + "logits/rejected": -1.0569286346435547, + "logps/chosen": -650.4686279296875, + "logps/rejected": -801.4287109375, + "loss": 0.4244, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.111543655395508, + "rewards/margins": 1.6189815998077393, + "rewards/rejected": -5.730525970458984, + "step": 231 + }, + { + "epoch": 0.49585893668180603, + "grad_norm": 20.212574699658393, + "learning_rate": 4.1506906250702017e-07, + "logits/chosen": -0.9895690083503723, + "logits/rejected": -0.9893757700920105, + "logps/chosen": -653.65673828125, + "logps/rejected": -774.2328491210938, + "loss": 0.4351, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.012841701507568, + "rewards/margins": 1.2045834064483643, + "rewards/rejected": -5.217425346374512, + "step": 232 + }, + { + "epoch": 0.49799625968474487, + "grad_norm": 19.25316743472589, + "learning_rate": 4.124949131795228e-07, + "logits/chosen": -0.9950495362281799, + "logits/rejected": -0.9298911690711975, + "logps/chosen": -593.9339599609375, + "logps/rejected": -793.3387451171875, + "loss": 0.3809, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.7703654766082764, + "rewards/margins": 1.754205346107483, + "rewards/rejected": -5.524570941925049, + "step": 233 + }, + { + "epoch": 0.5001335826876837, + "grad_norm": 18.738876938950654, + "learning_rate": 4.0991726726929135e-07, + "logits/chosen": -1.06260085105896, + "logits/rejected": -1.1126683950424194, + "logps/chosen": -576.23583984375, + "logps/rejected": -658.715576171875, + "loss": 0.5016, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.6776123046875, + "rewards/margins": 0.8450465202331543, + "rewards/rejected": -4.5226593017578125, + "step": 234 + }, + { + "epoch": 0.5022709056906225, + "grad_norm": 15.19487228526955, + "learning_rate": 4.073362689952982e-07, + "logits/chosen": -0.895923912525177, + "logits/rejected": -0.9245651364326477, + "logps/chosen": -693.7532348632812, + "logps/rejected": -714.86865234375, + "loss": 0.3918, + "rewards/accuracies": 0.625, + "rewards/chosen": -4.3402838706970215, + "rewards/margins": 0.3496522903442383, + "rewards/rejected": -4.68993616104126, + "step": 235 + }, + { + "epoch": 0.5044082286935613, + "grad_norm": 23.63383518264876, + "learning_rate": 4.047520627640808e-07, + "logits/chosen": -1.0637860298156738, + "logits/rejected": -1.0301114320755005, + "logps/chosen": -648.5076293945312, + "logps/rejected": -772.224609375, + "loss": 0.4001, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.6958110332489014, + "rewards/margins": 1.3828696012496948, + "rewards/rejected": -5.078680515289307, + "step": 236 + }, + { + "epoch": 0.5065455516965002, + "grad_norm": 26.41816558524286, + "learning_rate": 4.02164793161661e-07, + "logits/chosen": -0.9920711517333984, + "logits/rejected": -0.9205961227416992, + "logps/chosen": -606.1458129882812, + "logps/rejected": -882.5693969726562, + "loss": 0.4818, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.58052396774292, + "rewards/margins": 0.41025519371032715, + "rewards/rejected": -3.9907796382904053, + "step": 237 + }, + { + "epoch": 0.508682874699439, + "grad_norm": 15.74897212135201, + "learning_rate": 3.9957460494545616e-07, + "logits/chosen": -0.9549577236175537, + "logits/rejected": -0.9211697578430176, + "logps/chosen": -669.394287109375, + "logps/rejected": -709.4468994140625, + "loss": 0.4783, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.0179338455200195, + "rewards/margins": 0.4471871554851532, + "rewards/rejected": -4.465120792388916, + "step": 238 + }, + { + "epoch": 0.5108201977023777, + "grad_norm": 16.016701961339525, + "learning_rate": 3.969816430361794e-07, + "logits/chosen": -1.0792049169540405, + "logits/rejected": -0.9834136366844177, + "logps/chosen": -673.969970703125, + "logps/rejected": -964.8361206054688, + "loss": 0.3911, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.139585018157959, + "rewards/margins": 2.691437244415283, + "rewards/rejected": -6.831022262573242, + "step": 239 + }, + { + "epoch": 0.5129575207053166, + "grad_norm": 17.928342726480253, + "learning_rate": 3.9438605250973193e-07, + "logits/chosen": -0.9689183831214905, + "logits/rejected": -0.9091028571128845, + "logps/chosen": -529.1758422851562, + "logps/rejected": -694.4168090820312, + "loss": 0.4055, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.4839565753936768, + "rewards/margins": 1.5238869190216064, + "rewards/rejected": -5.007843494415283, + "step": 240 + }, + { + "epoch": 0.5150948437082554, + "grad_norm": 13.751574315942849, + "learning_rate": 3.9178797858908544e-07, + "logits/chosen": -0.9920436143875122, + "logits/rejected": -0.9840397834777832, + "logps/chosen": -706.0386352539062, + "logps/rejected": -849.04345703125, + "loss": 0.3468, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.169567108154297, + "rewards/margins": 1.2529165744781494, + "rewards/rejected": -5.422484397888184, + "step": 241 + }, + { + "epoch": 0.5172321667111942, + "grad_norm": 13.92465051011581, + "learning_rate": 3.891875666361577e-07, + "logits/chosen": -0.9852272868156433, + "logits/rejected": -0.9285619258880615, + "logps/chosen": -515.2431030273438, + "logps/rejected": -713.544189453125, + "loss": 0.3992, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.9760236740112305, + "rewards/margins": 1.73358952999115, + "rewards/rejected": -4.709612846374512, + "step": 242 + }, + { + "epoch": 0.5193694897141331, + "grad_norm": 15.781523786771364, + "learning_rate": 3.865849621436787e-07, + "logits/chosen": -1.0415364503860474, + "logits/rejected": -1.0579416751861572, + "logps/chosen": -614.417236328125, + "logps/rejected": -755.8591918945312, + "loss": 0.4447, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.9967360496520996, + "rewards/margins": 1.3200428485870361, + "rewards/rejected": -5.316779136657715, + "step": 243 + }, + { + "epoch": 0.5215068127170719, + "grad_norm": 13.998215826829304, + "learning_rate": 3.839803107270509e-07, + "logits/chosen": -0.8844782710075378, + "logits/rejected": -0.7867937088012695, + "logps/chosen": -551.641845703125, + "logps/rejected": -744.5933227539062, + "loss": 0.4074, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.376842737197876, + "rewards/margins": 1.7052807807922363, + "rewards/rejected": -5.082123756408691, + "step": 244 + }, + { + "epoch": 0.5236441357200107, + "grad_norm": 14.905532531142882, + "learning_rate": 3.813737581162017e-07, + "logits/chosen": -1.0912367105484009, + "logits/rejected": -1.0676392316818237, + "logps/chosen": -624.3543701171875, + "logps/rejected": -729.902099609375, + "loss": 0.413, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.5566658973693848, + "rewards/margins": 1.0893948078155518, + "rewards/rejected": -4.646060943603516, + "step": 245 + }, + { + "epoch": 0.5257814587229495, + "grad_norm": 13.609157264860004, + "learning_rate": 3.7876545014743065e-07, + "logits/chosen": -0.898338258266449, + "logits/rejected": -0.9181092381477356, + "logps/chosen": -624.335205078125, + "logps/rejected": -792.426025390625, + "loss": 0.4068, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.840670108795166, + "rewards/margins": 1.4304872751235962, + "rewards/rejected": -5.2711567878723145, + "step": 246 + }, + { + "epoch": 0.5279187817258884, + "grad_norm": 15.394561748839626, + "learning_rate": 3.761555327552485e-07, + "logits/chosen": -1.0056723356246948, + "logits/rejected": -1.0341317653656006, + "logps/chosen": -591.2755737304688, + "logps/rejected": -716.7362060546875, + "loss": 0.3879, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.308074951171875, + "rewards/margins": 1.277435302734375, + "rewards/rejected": -4.585509777069092, + "step": 247 + }, + { + "epoch": 0.5300561047288271, + "grad_norm": 16.757770907784614, + "learning_rate": 3.735441519642139e-07, + "logits/chosen": -1.1397889852523804, + "logits/rejected": -1.1101233959197998, + "logps/chosen": -517.30810546875, + "logps/rejected": -605.835205078125, + "loss": 0.4748, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.128363847732544, + "rewards/margins": 0.9252746105194092, + "rewards/rejected": -4.053638935089111, + "step": 248 + }, + { + "epoch": 0.5321934277317659, + "grad_norm": 14.363837462668146, + "learning_rate": 3.70931453880762e-07, + "logits/chosen": -0.9677891135215759, + "logits/rejected": -0.9882450699806213, + "logps/chosen": -697.7656860351562, + "logps/rejected": -846.583740234375, + "loss": 0.4142, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.01542329788208, + "rewards/margins": 1.4604833126068115, + "rewards/rejected": -5.4759063720703125, + "step": 249 + }, + { + "epoch": 0.5343307507347048, + "grad_norm": 18.905238772507477, + "learning_rate": 3.6831758468503035e-07, + "logits/chosen": -1.1917240619659424, + "logits/rejected": -1.1658238172531128, + "logps/chosen": -629.4384155273438, + "logps/rejected": -744.068603515625, + "loss": 0.5198, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.910032272338867, + "rewards/margins": 1.0619316101074219, + "rewards/rejected": -4.971964359283447, + "step": 250 + }, + { + "epoch": 0.5364680737376436, + "grad_norm": 13.775541697952745, + "learning_rate": 3.6570269062268025e-07, + "logits/chosen": -0.8680559992790222, + "logits/rejected": -0.8241602182388306, + "logps/chosen": -813.6458740234375, + "logps/rejected": -992.6861572265625, + "loss": 0.3533, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.124260425567627, + "rewards/margins": 1.764125108718872, + "rewards/rejected": -6.88838529586792, + "step": 251 + }, + { + "epoch": 0.5386053967405824, + "grad_norm": 22.37517195362321, + "learning_rate": 3.6308691799671404e-07, + "logits/chosen": -0.7893460392951965, + "logits/rejected": -0.8846604824066162, + "logps/chosen": -585.7125854492188, + "logps/rejected": -700.5401611328125, + "loss": 0.4298, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.845160961151123, + "rewards/margins": 1.1050822734832764, + "rewards/rejected": -4.9502434730529785, + "step": 252 + }, + { + "epoch": 0.5407427197435213, + "grad_norm": 20.261154485800315, + "learning_rate": 3.604704131592895e-07, + "logits/chosen": -1.1260894536972046, + "logits/rejected": -1.116862177848816, + "logps/chosen": -766.9825439453125, + "logps/rejected": -869.4150390625, + "loss": 0.4666, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.645950794219971, + "rewards/margins": 1.14671790599823, + "rewards/rejected": -5.79266881942749, + "step": 253 + }, + { + "epoch": 0.5428800427464601, + "grad_norm": 24.637498347289263, + "learning_rate": 3.578533225035318e-07, + "logits/chosen": -1.0664944648742676, + "logits/rejected": -1.0649199485778809, + "logps/chosen": -772.8311157226562, + "logps/rejected": -995.858642578125, + "loss": 0.3884, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.625110149383545, + "rewards/margins": 2.349405527114868, + "rewards/rejected": -6.974515914916992, + "step": 254 + }, + { + "epoch": 0.5450173657493989, + "grad_norm": 18.22340364830364, + "learning_rate": 3.552357924553423e-07, + "logits/chosen": -0.9930300712585449, + "logits/rejected": -1.0258715152740479, + "logps/chosen": -745.672119140625, + "logps/rejected": -877.061279296875, + "loss": 0.4139, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.158998489379883, + "rewards/margins": 1.6085114479064941, + "rewards/rejected": -5.767509937286377, + "step": 255 + }, + { + "epoch": 0.5471546887523377, + "grad_norm": 16.289391048607587, + "learning_rate": 3.526179694652065e-07, + "logits/chosen": -1.0990363359451294, + "logits/rejected": -1.0807216167449951, + "logps/chosen": -646.4122314453125, + "logps/rejected": -707.5439453125, + "loss": 0.4572, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.093816757202148, + "rewards/margins": 0.4628375768661499, + "rewards/rejected": -4.556654453277588, + "step": 256 + }, + { + "epoch": 0.5492920117552765, + "grad_norm": 20.631321302576914, + "learning_rate": 3.5e-07, + "logits/chosen": -1.1180474758148193, + "logits/rejected": -1.0840083360671997, + "logps/chosen": -714.1011352539062, + "logps/rejected": -901.5518798828125, + "loss": 0.5129, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.527349948883057, + "rewards/margins": 1.6243555545806885, + "rewards/rejected": -6.151705265045166, + "step": 257 + }, + { + "epoch": 0.5514293347582153, + "grad_norm": 17.30980385185334, + "learning_rate": 3.4738203053479345e-07, + "logits/chosen": -1.1147381067276, + "logits/rejected": -1.0951800346374512, + "logps/chosen": -671.677978515625, + "logps/rejected": -869.009033203125, + "loss": 0.3768, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.159421443939209, + "rewards/margins": 1.6203829050064087, + "rewards/rejected": -5.77980375289917, + "step": 258 + }, + { + "epoch": 0.5535666577611541, + "grad_norm": 16.77670843921571, + "learning_rate": 3.447642075446578e-07, + "logits/chosen": -1.027138113975525, + "logits/rejected": -1.0414177179336548, + "logps/chosen": -600.3823852539062, + "logps/rejected": -688.4371337890625, + "loss": 0.3932, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.747048854827881, + "rewards/margins": 0.837365448474884, + "rewards/rejected": -4.584414482116699, + "step": 259 + }, + { + "epoch": 0.555703980764093, + "grad_norm": 13.731293304469197, + "learning_rate": 3.421466774964683e-07, + "logits/chosen": -1.060163140296936, + "logits/rejected": -1.1123335361480713, + "logps/chosen": -769.50244140625, + "logps/rejected": -878.5049438476562, + "loss": 0.4123, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.471192359924316, + "rewards/margins": 1.6601918935775757, + "rewards/rejected": -6.131383895874023, + "step": 260 + }, + { + "epoch": 0.5578413037670318, + "grad_norm": 21.94893337829568, + "learning_rate": 3.395295868407106e-07, + "logits/chosen": -0.7995742559432983, + "logits/rejected": -0.7949234247207642, + "logps/chosen": -540.7459716796875, + "logps/rejected": -606.6634521484375, + "loss": 0.4368, + "rewards/accuracies": 0.5625, + "rewards/chosen": -3.475754976272583, + "rewards/margins": 0.5799713134765625, + "rewards/rejected": -4.055726051330566, + "step": 261 + }, + { + "epoch": 0.5599786267699706, + "grad_norm": 18.73889178987264, + "learning_rate": 3.3691308200328604e-07, + "logits/chosen": -0.8902988433837891, + "logits/rejected": -0.9296258687973022, + "logps/chosen": -738.5679931640625, + "logps/rejected": -910.6119384765625, + "loss": 0.4267, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.609591007232666, + "rewards/margins": 1.8606202602386475, + "rewards/rejected": -6.470211029052734, + "step": 262 + }, + { + "epoch": 0.5621159497729095, + "grad_norm": 17.495724524128903, + "learning_rate": 3.342973093773199e-07, + "logits/chosen": -1.0284425020217896, + "logits/rejected": -1.0994598865509033, + "logps/chosen": -763.5560302734375, + "logps/rejected": -916.3192749023438, + "loss": 0.4193, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.318557262420654, + "rewards/margins": 1.6308737993240356, + "rewards/rejected": -5.949431419372559, + "step": 263 + }, + { + "epoch": 0.5642532727758482, + "grad_norm": 21.8003372321031, + "learning_rate": 3.316824153149696e-07, + "logits/chosen": -0.9204785823822021, + "logits/rejected": -0.9681702852249146, + "logps/chosen": -642.7691650390625, + "logps/rejected": -805.9739990234375, + "loss": 0.4396, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.044567108154297, + "rewards/margins": 1.6253376007080078, + "rewards/rejected": -5.669903755187988, + "step": 264 + }, + { + "epoch": 0.566390595778787, + "grad_norm": 29.221196391296576, + "learning_rate": 3.29068546119238e-07, + "logits/chosen": -0.8801388740539551, + "logits/rejected": -0.8704509139060974, + "logps/chosen": -707.189453125, + "logps/rejected": -804.7384033203125, + "loss": 0.597, + "rewards/accuracies": 0.625, + "rewards/chosen": -4.639124870300293, + "rewards/margins": 1.1081290245056152, + "rewards/rejected": -5.74725341796875, + "step": 265 + }, + { + "epoch": 0.5685279187817259, + "grad_norm": 21.440238859390956, + "learning_rate": 3.2645584803578614e-07, + "logits/chosen": -0.9265174865722656, + "logits/rejected": -0.9847102761268616, + "logps/chosen": -483.4872131347656, + "logps/rejected": -527.4281005859375, + "loss": 0.44, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.243124008178711, + "rewards/margins": 0.43065446615219116, + "rewards/rejected": -3.6737782955169678, + "step": 266 + }, + { + "epoch": 0.5706652417846647, + "grad_norm": 15.539782944871728, + "learning_rate": 3.238444672447515e-07, + "logits/chosen": -0.8670752048492432, + "logits/rejected": -0.8761327862739563, + "logps/chosen": -669.8744506835938, + "logps/rejected": -792.420166015625, + "loss": 0.4026, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.5057759284973145, + "rewards/margins": 1.2092504501342773, + "rewards/rejected": -5.715025901794434, + "step": 267 + }, + { + "epoch": 0.5728025647876035, + "grad_norm": 15.529372167608297, + "learning_rate": 3.2123454985256943e-07, + "logits/chosen": -0.9794555902481079, + "logits/rejected": -0.9547824859619141, + "logps/chosen": -610.2971801757812, + "logps/rejected": -825.6856079101562, + "loss": 0.3279, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.7575111389160156, + "rewards/margins": 2.0598204135894775, + "rewards/rejected": -5.8173322677612305, + "step": 268 + }, + { + "epoch": 0.5749398877905424, + "grad_norm": 22.72130516871319, + "learning_rate": 3.186262418837983e-07, + "logits/chosen": -1.082902431488037, + "logits/rejected": -1.1030265092849731, + "logps/chosen": -545.2824096679688, + "logps/rejected": -609.6895141601562, + "loss": 0.5641, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.404395580291748, + "rewards/margins": 0.5698573589324951, + "rewards/rejected": -3.974252939224243, + "step": 269 + }, + { + "epoch": 0.5770772107934812, + "grad_norm": 16.252579751792183, + "learning_rate": 3.1601968927294914e-07, + "logits/chosen": -0.9424474239349365, + "logits/rejected": -0.9634624123573303, + "logps/chosen": -599.9176025390625, + "logps/rejected": -702.5125732421875, + "loss": 0.4418, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.833646774291992, + "rewards/margins": 0.9874881505966187, + "rewards/rejected": -4.821134567260742, + "step": 270 + }, + { + "epoch": 0.57921453379642, + "grad_norm": 15.206487367829114, + "learning_rate": 3.134150378563213e-07, + "logits/chosen": -0.9836793541908264, + "logits/rejected": -0.9680826663970947, + "logps/chosen": -637.9475708007812, + "logps/rejected": -767.5353393554688, + "loss": 0.3601, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.120051383972168, + "rewards/margins": 1.235574722290039, + "rewards/rejected": -5.355626106262207, + "step": 271 + }, + { + "epoch": 0.5813518567993589, + "grad_norm": 21.078417332237123, + "learning_rate": 3.1081243336384226e-07, + "logits/chosen": -0.9045091271400452, + "logits/rejected": -0.9048910140991211, + "logps/chosen": -615.5716552734375, + "logps/rejected": -752.4681396484375, + "loss": 0.3887, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.629241704940796, + "rewards/margins": 1.4587962627410889, + "rewards/rejected": -5.088037490844727, + "step": 272 + }, + { + "epoch": 0.5834891798022976, + "grad_norm": 17.81077343546014, + "learning_rate": 3.0821202141091453e-07, + "logits/chosen": -1.1009454727172852, + "logits/rejected": -0.9980039596557617, + "logps/chosen": -591.7613525390625, + "logps/rejected": -779.107421875, + "loss": 0.4669, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.549128770828247, + "rewards/margins": 1.750599980354309, + "rewards/rejected": -5.299729824066162, + "step": 273 + }, + { + "epoch": 0.5856265028052364, + "grad_norm": 19.256523288494076, + "learning_rate": 3.0561394749026814e-07, + "logits/chosen": -1.008500576019287, + "logits/rejected": -0.9249081611633301, + "logps/chosen": -631.9851684570312, + "logps/rejected": -834.7702026367188, + "loss": 0.4028, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.780165195465088, + "rewards/margins": 1.7658891677856445, + "rewards/rejected": -5.546054363250732, + "step": 274 + }, + { + "epoch": 0.5877638258081752, + "grad_norm": 21.051631953087565, + "learning_rate": 3.030183569638207e-07, + "logits/chosen": -1.1362390518188477, + "logits/rejected": -1.0792036056518555, + "logps/chosen": -576.5594482421875, + "logps/rejected": -694.7816162109375, + "loss": 0.4611, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.658750534057617, + "rewards/margins": 1.0495951175689697, + "rewards/rejected": -4.708345890045166, + "step": 275 + }, + { + "epoch": 0.5899011488111141, + "grad_norm": 15.76447884471548, + "learning_rate": 3.0042539505454386e-07, + "logits/chosen": -0.9091801643371582, + "logits/rejected": -0.9027244448661804, + "logps/chosen": -592.4364013671875, + "logps/rejected": -717.2314453125, + "loss": 0.3772, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.7294676303863525, + "rewards/margins": 1.2672703266143799, + "rewards/rejected": -4.996737957000732, + "step": 276 + }, + { + "epoch": 0.5920384718140529, + "grad_norm": 14.607600516553031, + "learning_rate": 2.978352068383389e-07, + "logits/chosen": -0.9434270858764648, + "logits/rejected": -0.9658694267272949, + "logps/chosen": -426.6009216308594, + "logps/rejected": -538.635009765625, + "loss": 0.3716, + "rewards/accuracies": 0.8125, + "rewards/chosen": -2.5574347972869873, + "rewards/margins": 1.0464235544204712, + "rewards/rejected": -3.603858470916748, + "step": 277 + }, + { + "epoch": 0.5941757948169917, + "grad_norm": 20.680956313480134, + "learning_rate": 2.952479372359191e-07, + "logits/chosen": -0.9002370238304138, + "logits/rejected": -0.9020406007766724, + "logps/chosen": -564.5108642578125, + "logps/rejected": -660.9511108398438, + "loss": 0.3884, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.561006784439087, + "rewards/margins": 0.8000065088272095, + "rewards/rejected": -4.361013412475586, + "step": 278 + }, + { + "epoch": 0.5963131178199306, + "grad_norm": 27.6993585691743, + "learning_rate": 2.926637310047018e-07, + "logits/chosen": -1.1155834197998047, + "logits/rejected": -1.105792760848999, + "logps/chosen": -596.0205078125, + "logps/rejected": -664.81640625, + "loss": 0.3998, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.8352932929992676, + "rewards/margins": 0.8193560838699341, + "rewards/rejected": -4.654649257659912, + "step": 279 + }, + { + "epoch": 0.5984504408228694, + "grad_norm": 18.916329978828376, + "learning_rate": 2.900827327307087e-07, + "logits/chosen": -1.2077603340148926, + "logits/rejected": -1.0835894346237183, + "logps/chosen": -778.5531005859375, + "logps/rejected": -890.9183349609375, + "loss": 0.4658, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.6639227867126465, + "rewards/margins": 1.0659689903259277, + "rewards/rejected": -5.729891777038574, + "step": 280 + }, + { + "epoch": 0.6005877638258081, + "grad_norm": 18.365489297240394, + "learning_rate": 2.8750508682047717e-07, + "logits/chosen": -1.1601464748382568, + "logits/rejected": -1.1277116537094116, + "logps/chosen": -677.4540405273438, + "logps/rejected": -838.7645874023438, + "loss": 0.4918, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.8409969806671143, + "rewards/margins": 1.480958342552185, + "rewards/rejected": -5.321955680847168, + "step": 281 + }, + { + "epoch": 0.602725086828747, + "grad_norm": 20.141526963645916, + "learning_rate": 2.849309374929799e-07, + "logits/chosen": -1.1542294025421143, + "logits/rejected": -1.1290372610092163, + "logps/chosen": -638.0623779296875, + "logps/rejected": -759.2158203125, + "loss": 0.4268, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.9765753746032715, + "rewards/margins": 1.2076830863952637, + "rewards/rejected": -5.184258460998535, + "step": 282 + }, + { + "epoch": 0.6048624098316858, + "grad_norm": 17.776205208549275, + "learning_rate": 2.823604287715564e-07, + "logits/chosen": -0.9990053772926331, + "logits/rejected": -1.038635015487671, + "logps/chosen": -641.4173583984375, + "logps/rejected": -752.838623046875, + "loss": 0.3768, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.788572072982788, + "rewards/margins": 1.1508256196975708, + "rewards/rejected": -4.939397811889648, + "step": 283 + }, + { + "epoch": 0.6069997328346246, + "grad_norm": 26.51717001409862, + "learning_rate": 2.797937044758545e-07, + "logits/chosen": -0.9398884177207947, + "logits/rejected": -0.9421570897102356, + "logps/chosen": -616.0810546875, + "logps/rejected": -697.7490844726562, + "loss": 0.4983, + "rewards/accuracies": 0.625, + "rewards/chosen": -4.009638786315918, + "rewards/margins": 0.752707302570343, + "rewards/rejected": -4.762345790863037, + "step": 284 + }, + { + "epoch": 0.6091370558375635, + "grad_norm": 35.30303590344862, + "learning_rate": 2.772309082137843e-07, + "logits/chosen": -1.085971474647522, + "logits/rejected": -1.0797044038772583, + "logps/chosen": -635.1014404296875, + "logps/rejected": -687.0366821289062, + "loss": 0.5866, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.941286325454712, + "rewards/margins": 0.7452540397644043, + "rewards/rejected": -4.686540603637695, + "step": 285 + }, + { + "epoch": 0.6112743788405023, + "grad_norm": 26.601600287549388, + "learning_rate": 2.7467218337348233e-07, + "logits/chosen": -0.8175097703933716, + "logits/rejected": -0.7498407959938049, + "logps/chosen": -589.5838623046875, + "logps/rejected": -743.729248046875, + "loss": 0.4928, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.852874279022217, + "rewards/margins": 1.4163994789123535, + "rewards/rejected": -5.2692742347717285, + "step": 286 + }, + { + "epoch": 0.6134117018434411, + "grad_norm": 17.687526214496195, + "learning_rate": 2.7211767311529e-07, + "logits/chosen": -0.8146362900733948, + "logits/rejected": -0.8423186540603638, + "logps/chosen": -607.890869140625, + "logps/rejected": -708.3568115234375, + "loss": 0.4853, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.738939046859741, + "rewards/margins": 0.9543510675430298, + "rewards/rejected": -4.693289756774902, + "step": 287 + }, + { + "epoch": 0.6155490248463799, + "grad_norm": 18.930801664854705, + "learning_rate": 2.695675203637432e-07, + "logits/chosen": -1.0421204566955566, + "logits/rejected": -1.0792558193206787, + "logps/chosen": -602.5674438476562, + "logps/rejected": -736.7977905273438, + "loss": 0.4063, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.712022304534912, + "rewards/margins": 1.2383270263671875, + "rewards/rejected": -4.9503493309021, + "step": 288 + }, + { + "epoch": 0.6176863478493188, + "grad_norm": 17.90358421564856, + "learning_rate": 2.670218677995759e-07, + "logits/chosen": -1.0456268787384033, + "logits/rejected": -0.9645753502845764, + "logps/chosen": -618.1510009765625, + "logps/rejected": -819.269287109375, + "loss": 0.4206, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.5795562267303467, + "rewards/margins": 2.006627321243286, + "rewards/rejected": -5.586184024810791, + "step": 289 + }, + { + "epoch": 0.6198236708522575, + "grad_norm": 23.380000713489068, + "learning_rate": 2.6448085785173703e-07, + "logits/chosen": -1.005885124206543, + "logits/rejected": -0.964793860912323, + "logps/chosen": -665.0631103515625, + "logps/rejected": -754.4349975585938, + "loss": 0.4141, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.058958053588867, + "rewards/margins": 0.8748332262039185, + "rewards/rejected": -4.933791160583496, + "step": 290 + }, + { + "epoch": 0.6219609938551963, + "grad_norm": 16.59292554666212, + "learning_rate": 2.6194463268942203e-07, + "logits/chosen": -1.0780235528945923, + "logits/rejected": -1.097104787826538, + "logps/chosen": -590.5000610351562, + "logps/rejected": -796.20166015625, + "loss": 0.3928, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.4696645736694336, + "rewards/margins": 1.882230281829834, + "rewards/rejected": -5.351894855499268, + "step": 291 + }, + { + "epoch": 0.6240983168581352, + "grad_norm": 14.429424103488177, + "learning_rate": 2.594133342141177e-07, + "logits/chosen": -0.9270427227020264, + "logits/rejected": -0.8782027959823608, + "logps/chosen": -498.6771545410156, + "logps/rejected": -686.7008056640625, + "loss": 0.3666, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.730340003967285, + "rewards/margins": 1.7681416273117065, + "rewards/rejected": -4.498481273651123, + "step": 292 + }, + { + "epoch": 0.626235639861074, + "grad_norm": 21.064661843353115, + "learning_rate": 2.568871040516637e-07, + "logits/chosen": -1.0022443532943726, + "logits/rejected": -0.9993023872375488, + "logps/chosen": -685.757080078125, + "logps/rejected": -779.1624755859375, + "loss": 0.4839, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.1098246574401855, + "rewards/margins": 0.944150984287262, + "rewards/rejected": -5.053975582122803, + "step": 293 + }, + { + "epoch": 0.6283729628640128, + "grad_norm": 19.087071179960915, + "learning_rate": 2.54366083544328e-07, + "logits/chosen": -0.8884956240653992, + "logits/rejected": -0.9064831137657166, + "logps/chosen": -571.4553833007812, + "logps/rejected": -674.0166625976562, + "loss": 0.504, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.6071152687072754, + "rewards/margins": 0.9564936757087708, + "rewards/rejected": -4.563608646392822, + "step": 294 + }, + { + "epoch": 0.6305102858669517, + "grad_norm": 21.794412344839774, + "learning_rate": 2.5185041374289907e-07, + "logits/chosen": -1.1455574035644531, + "logits/rejected": -1.1415196657180786, + "logps/chosen": -497.8589782714844, + "logps/rejected": -751.036376953125, + "loss": 0.4041, + "rewards/accuracies": 0.9375, + "rewards/chosen": -2.744825839996338, + "rewards/margins": 2.1894543170928955, + "rewards/rejected": -4.934279918670654, + "step": 295 + }, + { + "epoch": 0.6326476088698905, + "grad_norm": 19.554010929778244, + "learning_rate": 2.4934023539879395e-07, + "logits/chosen": -0.979045569896698, + "logits/rejected": -0.9661969542503357, + "logps/chosen": -601.6738891601562, + "logps/rejected": -756.327392578125, + "loss": 0.3696, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.3881332874298096, + "rewards/margins": 1.3837690353393555, + "rewards/rejected": -4.771902561187744, + "step": 296 + }, + { + "epoch": 0.6347849318728293, + "grad_norm": 20.678635374709422, + "learning_rate": 2.468356889561835e-07, + "logits/chosen": -1.0623462200164795, + "logits/rejected": -1.0434157848358154, + "logps/chosen": -569.0560302734375, + "logps/rejected": -698.5628051757812, + "loss": 0.3802, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.12467622756958, + "rewards/margins": 1.3073335886001587, + "rewards/rejected": -4.432010173797607, + "step": 297 + }, + { + "epoch": 0.6369222548757681, + "grad_norm": 15.887777705629667, + "learning_rate": 2.443369145441343e-07, + "logits/chosen": -1.0596822500228882, + "logits/rejected": -1.044713020324707, + "logps/chosen": -531.6007080078125, + "logps/rejected": -636.859130859375, + "loss": 0.3491, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.2483232021331787, + "rewards/margins": 1.0054796934127808, + "rewards/rejected": -4.253802299499512, + "step": 298 + }, + { + "epoch": 0.6390595778787069, + "grad_norm": 20.882809623209052, + "learning_rate": 2.418440519687684e-07, + "logits/chosen": -1.0757739543914795, + "logits/rejected": -1.0567525625228882, + "logps/chosen": -511.7173156738281, + "logps/rejected": -668.195556640625, + "loss": 0.4593, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.052992820739746, + "rewards/margins": 1.4922919273376465, + "rewards/rejected": -4.545284271240234, + "step": 299 + }, + { + "epoch": 0.6411969008816457, + "grad_norm": 14.618996262053995, + "learning_rate": 2.393572407054416e-07, + "logits/chosen": -0.8958306312561035, + "logits/rejected": -0.8867323398590088, + "logps/chosen": -659.5025634765625, + "logps/rejected": -834.076171875, + "loss": 0.3802, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.8700366020202637, + "rewards/margins": 1.7134443521499634, + "rewards/rejected": -5.5834808349609375, + "step": 300 + }, + { + "epoch": 0.6411969008816457, + "eval_logits/chosen": -1.1104539632797241, + "eval_logits/rejected": -1.130950927734375, + "eval_logps/chosen": -652.87158203125, + "eval_logps/rejected": -784.0881958007812, + "eval_loss": 0.40694576501846313, + "eval_rewards/accuracies": 0.8427419066429138, + "eval_rewards/chosen": -3.7974159717559814, + "eval_rewards/margins": 1.3731279373168945, + "eval_rewards/rejected": -5.170544147491455, + "eval_runtime": 127.0172, + "eval_samples_per_second": 15.439, + "eval_steps_per_second": 0.488, + "step": 300 + }, + { + "epoch": 0.6433342238845846, + "grad_norm": 19.43800281730569, + "learning_rate": 2.3687661989093904e-07, + "logits/chosen": -0.9887138605117798, + "logits/rejected": -1.0439525842666626, + "logps/chosen": -595.8880615234375, + "logps/rejected": -732.6058349609375, + "loss": 0.4198, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.636826753616333, + "rewards/margins": 1.4018120765686035, + "rewards/rejected": -5.038639068603516, + "step": 301 + }, + { + "epoch": 0.6454715468875234, + "grad_norm": 22.456716854896456, + "learning_rate": 2.3440232831569162e-07, + "logits/chosen": -1.0877864360809326, + "logits/rejected": -1.1251873970031738, + "logps/chosen": -665.487548828125, + "logps/rejected": -819.5171508789062, + "loss": 0.4536, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.9592628479003906, + "rewards/margins": 1.4847943782806396, + "rewards/rejected": -5.444056987762451, + "step": 302 + }, + { + "epoch": 0.6476088698904622, + "grad_norm": 16.0762099994994, + "learning_rate": 2.319345044160095e-07, + "logits/chosen": -1.2076233625411987, + "logits/rejected": -1.230367660522461, + "logps/chosen": -769.3408203125, + "logps/rejected": -1005.293701171875, + "loss": 0.4059, + "rewards/accuracies": 0.625, + "rewards/chosen": -4.611854076385498, + "rewards/margins": 2.47025465965271, + "rewards/rejected": -7.082108974456787, + "step": 303 + }, + { + "epoch": 0.649746192893401, + "grad_norm": 17.96340906701677, + "learning_rate": 2.2947328626633766e-07, + "logits/chosen": -1.058774709701538, + "logits/rejected": -1.0470080375671387, + "logps/chosen": -590.3370361328125, + "logps/rejected": -681.3898315429688, + "loss": 0.4649, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.8478586673736572, + "rewards/margins": 0.7897017002105713, + "rewards/rejected": -4.637560844421387, + "step": 304 + }, + { + "epoch": 0.6518835158963399, + "grad_norm": 22.719738681967296, + "learning_rate": 2.2701881157153e-07, + "logits/chosen": -1.1047711372375488, + "logits/rejected": -1.0974376201629639, + "logps/chosen": -653.9512329101562, + "logps/rejected": -839.6368408203125, + "loss": 0.3939, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.9316320419311523, + "rewards/margins": 1.9634764194488525, + "rewards/rejected": -5.895109176635742, + "step": 305 + }, + { + "epoch": 0.6540208388992786, + "grad_norm": 17.780668266782456, + "learning_rate": 2.245712176591449e-07, + "logits/chosen": -1.0736472606658936, + "logits/rejected": -1.0763237476348877, + "logps/chosen": -615.7779541015625, + "logps/rejected": -687.5728759765625, + "loss": 0.4217, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.957977533340454, + "rewards/margins": 0.773561954498291, + "rewards/rejected": -4.731539726257324, + "step": 306 + }, + { + "epoch": 0.6561581619022174, + "grad_norm": 23.74418488307736, + "learning_rate": 2.2213064147176174e-07, + "logits/chosen": -1.094491958618164, + "logits/rejected": -1.1138453483581543, + "logps/chosen": -725.0467529296875, + "logps/rejected": -822.287353515625, + "loss": 0.4067, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.600536823272705, + "rewards/margins": 1.1523770093917847, + "rewards/rejected": -5.752913951873779, + "step": 307 + }, + { + "epoch": 0.6582954849051563, + "grad_norm": 15.5978905581422, + "learning_rate": 2.196972195593192e-07, + "logits/chosen": -1.1050280332565308, + "logits/rejected": -1.0931296348571777, + "logps/chosen": -582.18115234375, + "logps/rejected": -741.2723388671875, + "loss": 0.3451, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.750410318374634, + "rewards/margins": 1.5357167720794678, + "rewards/rejected": -5.28612756729126, + "step": 308 + }, + { + "epoch": 0.6604328079080951, + "grad_norm": 17.049613660365612, + "learning_rate": 2.1727108807147506e-07, + "logits/chosen": -1.0725035667419434, + "logits/rejected": -1.1041122674942017, + "logps/chosen": -610.0225219726562, + "logps/rejected": -762.8485107421875, + "loss": 0.4524, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.9435532093048096, + "rewards/margins": 1.720339059829712, + "rewards/rejected": -5.663891792297363, + "step": 309 + }, + { + "epoch": 0.6625701309110339, + "grad_norm": 21.589679932544243, + "learning_rate": 2.148523827499888e-07, + "logits/chosen": -0.9776382446289062, + "logits/rejected": -1.0473860502243042, + "logps/chosen": -845.308837890625, + "logps/rejected": -1063.19482421875, + "loss": 0.4161, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.683403491973877, + "rewards/margins": 1.968895673751831, + "rewards/rejected": -7.652299404144287, + "step": 310 + }, + { + "epoch": 0.6647074539139728, + "grad_norm": 20.546655366561865, + "learning_rate": 2.1244123892112674e-07, + "logits/chosen": -0.971989095211029, + "logits/rejected": -0.9635407328605652, + "logps/chosen": -752.0794677734375, + "logps/rejected": -896.9614868164062, + "loss": 0.4412, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.961668968200684, + "rewards/margins": 1.418576717376709, + "rewards/rejected": -6.380245685577393, + "step": 311 + }, + { + "epoch": 0.6668447769169116, + "grad_norm": 16.64812094871957, + "learning_rate": 2.1003779148809072e-07, + "logits/chosen": -0.9267586469650269, + "logits/rejected": -0.9304214715957642, + "logps/chosen": -742.295166015625, + "logps/rejected": -886.9481201171875, + "loss": 0.3719, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.672271251678467, + "rewards/margins": 1.6211450099945068, + "rewards/rejected": -6.2934160232543945, + "step": 312 + }, + { + "epoch": 0.6689820999198504, + "grad_norm": 21.010619081649534, + "learning_rate": 2.0764217492346996e-07, + "logits/chosen": -1.0951707363128662, + "logits/rejected": -1.0666468143463135, + "logps/chosen": -790.2965087890625, + "logps/rejected": -910.8655395507812, + "loss": 0.4601, + "rewards/accuracies": 0.6875, + "rewards/chosen": -5.142488956451416, + "rewards/margins": 1.2062201499938965, + "rewards/rejected": -6.348708629608154, + "step": 313 + }, + { + "epoch": 0.6711194229227893, + "grad_norm": 19.84084802689762, + "learning_rate": 2.0525452326171784e-07, + "logits/chosen": -1.0453927516937256, + "logits/rejected": -1.0439743995666504, + "logps/chosen": -865.6875610351562, + "logps/rejected": -1150.561767578125, + "loss": 0.4036, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.613504409790039, + "rewards/margins": 2.756046772003174, + "rewards/rejected": -8.369551658630371, + "step": 314 + }, + { + "epoch": 0.673256745925728, + "grad_norm": 17.469930397973854, + "learning_rate": 2.028749700916521e-07, + "logits/chosen": -0.955051600933075, + "logits/rejected": -0.944666862487793, + "logps/chosen": -787.5214233398438, + "logps/rejected": -877.498779296875, + "loss": 0.403, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.070991039276123, + "rewards/margins": 0.8326537013053894, + "rewards/rejected": -5.903645038604736, + "step": 315 + }, + { + "epoch": 0.6753940689286668, + "grad_norm": 31.931815437246083, + "learning_rate": 2.0050364854898127e-07, + "logits/chosen": -1.066016674041748, + "logits/rejected": -1.063683032989502, + "logps/chosen": -740.451416015625, + "logps/rejected": -935.1329345703125, + "loss": 0.497, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.710479736328125, + "rewards/margins": 1.9114099740982056, + "rewards/rejected": -6.621890068054199, + "step": 316 + }, + { + "epoch": 0.6775313919316056, + "grad_norm": 23.195446430699608, + "learning_rate": 1.9814069130885467e-07, + "logits/chosen": -0.7954821586608887, + "logits/rejected": -0.8669359683990479, + "logps/chosen": -679.1461791992188, + "logps/rejected": -868.0947875976562, + "loss": 0.4159, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.395887851715088, + "rewards/margins": 2.0239951610565186, + "rewards/rejected": -6.4198832511901855, + "step": 317 + }, + { + "epoch": 0.6796687149345445, + "grad_norm": 27.100165283554862, + "learning_rate": 1.9578623057844064e-07, + "logits/chosen": -1.1223851442337036, + "logits/rejected": -1.1304949522018433, + "logps/chosen": -606.3875732421875, + "logps/rejected": -768.2408447265625, + "loss": 0.4085, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.690753936767578, + "rewards/margins": 1.4597177505493164, + "rewards/rejected": -5.1504716873168945, + "step": 318 + }, + { + "epoch": 0.6818060379374833, + "grad_norm": 19.59121701664144, + "learning_rate": 1.9344039808952853e-07, + "logits/chosen": -0.9872533082962036, + "logits/rejected": -0.934008002281189, + "logps/chosen": -740.6689453125, + "logps/rejected": -1047.7109375, + "loss": 0.3775, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.856841087341309, + "rewards/margins": 2.8627772331237793, + "rewards/rejected": -7.71961784362793, + "step": 319 + }, + { + "epoch": 0.6839433609404221, + "grad_norm": 20.56349471919662, + "learning_rate": 1.9110332509115864e-07, + "logits/chosen": -1.0536935329437256, + "logits/rejected": -1.111939549446106, + "logps/chosen": -713.4016723632812, + "logps/rejected": -897.5386962890625, + "loss": 0.4077, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.394659996032715, + "rewards/margins": 1.7375450134277344, + "rewards/rejected": -6.132205009460449, + "step": 320 + }, + { + "epoch": 0.686080683943361, + "grad_norm": 21.59919721276828, + "learning_rate": 1.8877514234227874e-07, + "logits/chosen": -0.8561320900917053, + "logits/rejected": -0.9341483116149902, + "logps/chosen": -669.7821044921875, + "logps/rejected": -811.287353515625, + "loss": 0.4224, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.155871391296387, + "rewards/margins": 1.4691696166992188, + "rewards/rejected": -5.6250410079956055, + "step": 321 + }, + { + "epoch": 0.6882180069462998, + "grad_norm": 19.329148607505996, + "learning_rate": 1.8645598010442828e-07, + "logits/chosen": -0.9411246180534363, + "logits/rejected": -0.9587753415107727, + "logps/chosen": -699.0866088867188, + "logps/rejected": -864.09375, + "loss": 0.3697, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.35011100769043, + "rewards/margins": 1.8306729793548584, + "rewards/rejected": -6.180783748626709, + "step": 322 + }, + { + "epoch": 0.6903553299492385, + "grad_norm": 16.796887314919882, + "learning_rate": 1.8414596813445047e-07, + "logits/chosen": -0.9905204772949219, + "logits/rejected": -1.024290680885315, + "logps/chosen": -622.2347412109375, + "logps/rejected": -708.5049438476562, + "loss": 0.407, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.9594295024871826, + "rewards/margins": 0.9213578104972839, + "rewards/rejected": -4.8807878494262695, + "step": 323 + }, + { + "epoch": 0.6924926529521774, + "grad_norm": 15.1418538158911, + "learning_rate": 1.8184523567723194e-07, + "logits/chosen": -1.0796232223510742, + "logits/rejected": -1.0750352144241333, + "logps/chosen": -628.67041015625, + "logps/rejected": -812.5968017578125, + "loss": 0.351, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.8600831031799316, + "rewards/margins": 1.7709476947784424, + "rewards/rejected": -5.631031036376953, + "step": 324 + }, + { + "epoch": 0.6946299759551162, + "grad_norm": 18.026973060407244, + "learning_rate": 1.7955391145847145e-07, + "logits/chosen": -0.9602807760238647, + "logits/rejected": -0.9538272619247437, + "logps/chosen": -533.8419189453125, + "logps/rejected": -605.8224487304688, + "loss": 0.338, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.626598596572876, + "rewards/margins": 0.6452375054359436, + "rewards/rejected": -4.271836280822754, + "step": 325 + }, + { + "epoch": 0.696767298958055, + "grad_norm": 20.22808705229649, + "learning_rate": 1.772721236774786e-07, + "logits/chosen": -0.8902896642684937, + "logits/rejected": -0.891946017742157, + "logps/chosen": -547.3910522460938, + "logps/rejected": -737.225830078125, + "loss": 0.3568, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.2871787548065186, + "rewards/margins": 1.8806804418563843, + "rewards/rejected": -5.167859077453613, + "step": 326 + }, + { + "epoch": 0.6989046219609939, + "grad_norm": 24.61898214966437, + "learning_rate": 1.7500000000000007e-07, + "logits/chosen": -1.0208594799041748, + "logits/rejected": -0.9971433877944946, + "logps/chosen": -628.2722778320312, + "logps/rejected": -766.7388916015625, + "loss": 0.4498, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.7911126613616943, + "rewards/margins": 1.395633339881897, + "rewards/rejected": -5.186745643615723, + "step": 327 + }, + { + "epoch": 0.7010419449639327, + "grad_norm": 23.202132151091234, + "learning_rate": 1.7273766755107707e-07, + "logits/chosen": -1.00026535987854, + "logits/rejected": -1.0049934387207031, + "logps/chosen": -618.7490234375, + "logps/rejected": -784.1082153320312, + "loss": 0.4044, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.9590413570404053, + "rewards/margins": 1.4626474380493164, + "rewards/rejected": -5.421689033508301, + "step": 328 + }, + { + "epoch": 0.7031792679668715, + "grad_norm": 22.040873223414177, + "learning_rate": 1.7048525290793284e-07, + "logits/chosen": -1.151913046836853, + "logits/rejected": -1.1811047792434692, + "logps/chosen": -676.8132934570312, + "logps/rejected": -835.91064453125, + "loss": 0.4432, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.948080062866211, + "rewards/margins": 1.632805347442627, + "rewards/rejected": -5.580885887145996, + "step": 329 + }, + { + "epoch": 0.7053165909698104, + "grad_norm": 16.489066064805076, + "learning_rate": 1.6824288209289077e-07, + "logits/chosen": -0.9843920469284058, + "logits/rejected": -0.9963368773460388, + "logps/chosen": -695.6856689453125, + "logps/rejected": -835.2234497070312, + "loss": 0.3754, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.025155067443848, + "rewards/margins": 1.4948614835739136, + "rewards/rejected": -5.520016193389893, + "step": 330 + }, + { + "epoch": 0.7074539139727491, + "grad_norm": 18.164643888570296, + "learning_rate": 1.660106805663233e-07, + "logits/chosen": -0.9996950626373291, + "logits/rejected": -1.0081021785736084, + "logps/chosen": -585.10009765625, + "logps/rejected": -739.9697265625, + "loss": 0.3477, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.4289603233337402, + "rewards/margins": 1.5738253593444824, + "rewards/rejected": -5.002785682678223, + "step": 331 + }, + { + "epoch": 0.7095912369756879, + "grad_norm": 18.104299917352034, + "learning_rate": 1.6378877321963224e-07, + "logits/chosen": -1.0804736614227295, + "logits/rejected": -1.0568981170654297, + "logps/chosen": -713.994140625, + "logps/rejected": -918.4098510742188, + "loss": 0.3097, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.44266414642334, + "rewards/margins": 1.9913928508758545, + "rewards/rejected": -6.434057712554932, + "step": 332 + }, + { + "epoch": 0.7117285599786267, + "grad_norm": 19.250777560445798, + "learning_rate": 1.6157728436826125e-07, + "logits/chosen": -1.0584830045700073, + "logits/rejected": -1.0436077117919922, + "logps/chosen": -645.3274536132812, + "logps/rejected": -729.302001953125, + "loss": 0.3813, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.107142925262451, + "rewards/margins": 0.8789306879043579, + "rewards/rejected": -4.9860734939575195, + "step": 333 + }, + { + "epoch": 0.7138658829815656, + "grad_norm": 18.363434689830413, + "learning_rate": 1.5937633774474053e-07, + "logits/chosen": -0.964330792427063, + "logits/rejected": -1.0107676982879639, + "logps/chosen": -681.0447387695312, + "logps/rejected": -893.3312377929688, + "loss": 0.4289, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.495457172393799, + "rewards/margins": 2.035775661468506, + "rewards/rejected": -6.531231880187988, + "step": 334 + }, + { + "epoch": 0.7160032059845044, + "grad_norm": 19.36554980671261, + "learning_rate": 1.5718605649176415e-07, + "logits/chosen": -1.08909273147583, + "logits/rejected": -1.029083013534546, + "logps/chosen": -606.0968017578125, + "logps/rejected": -801.4658203125, + "loss": 0.3958, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.555162191390991, + "rewards/margins": 1.8386735916137695, + "rewards/rejected": -5.393835544586182, + "step": 335 + }, + { + "epoch": 0.7181405289874432, + "grad_norm": 19.53200074466149, + "learning_rate": 1.5500656315529971e-07, + "logits/chosen": -0.9612129330635071, + "logits/rejected": -1.0220246315002441, + "logps/chosen": -689.0376586914062, + "logps/rejected": -871.2042846679688, + "loss": 0.4066, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.374889850616455, + "rewards/margins": 1.9528312683105469, + "rewards/rejected": -6.327720642089844, + "step": 336 + }, + { + "epoch": 0.7202778519903821, + "grad_norm": 16.525872426498882, + "learning_rate": 1.5283797967773228e-07, + "logits/chosen": -0.9871097803115845, + "logits/rejected": -1.0105241537094116, + "logps/chosen": -629.0919799804688, + "logps/rejected": -870.4136352539062, + "loss": 0.3259, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.831345558166504, + "rewards/margins": 2.3878016471862793, + "rewards/rejected": -6.219147205352783, + "step": 337 + }, + { + "epoch": 0.7224151749933209, + "grad_norm": 17.894860864055556, + "learning_rate": 1.506804273910418e-07, + "logits/chosen": -1.0389041900634766, + "logits/rejected": -0.9967656135559082, + "logps/chosen": -597.2425537109375, + "logps/rejected": -690.931884765625, + "loss": 0.3708, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.6145212650299072, + "rewards/margins": 1.04227876663208, + "rewards/rejected": -4.656799793243408, + "step": 338 + }, + { + "epoch": 0.7245524979962596, + "grad_norm": 24.07466390909236, + "learning_rate": 1.485340270100143e-07, + "logits/chosen": -0.8043147921562195, + "logits/rejected": -0.8566476106643677, + "logps/chosen": -721.7520751953125, + "logps/rejected": -766.68896484375, + "loss": 0.384, + "rewards/accuracies": 0.625, + "rewards/chosen": -4.88438081741333, + "rewards/margins": 0.5339137315750122, + "rewards/rejected": -5.418294906616211, + "step": 339 + }, + { + "epoch": 0.7266898209991985, + "grad_norm": 23.369533031738897, + "learning_rate": 1.4639889862548805e-07, + "logits/chosen": -1.1091313362121582, + "logits/rejected": -1.1004701852798462, + "logps/chosen": -672.283447265625, + "logps/rejected": -803.2059936523438, + "loss": 0.4195, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.266483306884766, + "rewards/margins": 1.2370930910110474, + "rewards/rejected": -5.503575801849365, + "step": 340 + }, + { + "epoch": 0.7288271440021373, + "grad_norm": 18.450003910880596, + "learning_rate": 1.4427516169763443e-07, + "logits/chosen": -0.9054552912712097, + "logits/rejected": -0.9048701524734497, + "logps/chosen": -747.39013671875, + "logps/rejected": -927.0717163085938, + "loss": 0.3928, + "rewards/accuracies": 0.75, + "rewards/chosen": -5.073859214782715, + "rewards/margins": 1.7517752647399902, + "rewards/rejected": -6.825634479522705, + "step": 341 + }, + { + "epoch": 0.7309644670050761, + "grad_norm": 23.60596160513393, + "learning_rate": 1.4216293504927448e-07, + "logits/chosen": -1.067272424697876, + "logits/rejected": -1.0798040628433228, + "logps/chosen": -676.371826171875, + "logps/rejected": -869.149169921875, + "loss": 0.379, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.598742961883545, + "rewards/margins": 1.6217763423919678, + "rewards/rejected": -6.220519065856934, + "step": 342 + }, + { + "epoch": 0.733101790008015, + "grad_norm": 21.614156690463627, + "learning_rate": 1.400623368592302e-07, + "logits/chosen": -0.9705036878585815, + "logits/rejected": -0.9520074725151062, + "logps/chosen": -678.7489013671875, + "logps/rejected": -816.748779296875, + "loss": 0.4281, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.580175876617432, + "rewards/margins": 1.3712455034255981, + "rewards/rejected": -5.951421737670898, + "step": 343 + }, + { + "epoch": 0.7352391130109538, + "grad_norm": 23.05746652103131, + "learning_rate": 1.3797348465571306e-07, + "logits/chosen": -0.9307321310043335, + "logits/rejected": -0.9591343402862549, + "logps/chosen": -746.435546875, + "logps/rejected": -1008.3950805664062, + "loss": 0.3567, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.570581912994385, + "rewards/margins": 2.517709255218506, + "rewards/rejected": -7.088291168212891, + "step": 344 + }, + { + "epoch": 0.7373764360138926, + "grad_norm": 18.10148284802051, + "learning_rate": 1.3589649530974773e-07, + "logits/chosen": -1.0869290828704834, + "logits/rejected": -1.0850858688354492, + "logps/chosen": -658.4027099609375, + "logps/rejected": -982.6063842773438, + "loss": 0.3781, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.2937188148498535, + "rewards/margins": 2.9649014472961426, + "rewards/rejected": -7.258620262145996, + "step": 345 + }, + { + "epoch": 0.7395137590168315, + "grad_norm": 23.41128054527833, + "learning_rate": 1.3383148502863378e-07, + "logits/chosen": -1.0312128067016602, + "logits/rejected": -1.1049691438674927, + "logps/chosen": -814.4356689453125, + "logps/rejected": -798.0429077148438, + "loss": 0.514, + "rewards/accuracies": 0.4375, + "rewards/chosen": -5.450601577758789, + "rewards/margins": 0.045907363295555115, + "rewards/rejected": -5.496509552001953, + "step": 346 + }, + { + "epoch": 0.7416510820197703, + "grad_norm": 18.50899288865318, + "learning_rate": 1.317785693494433e-07, + "logits/chosen": -1.0586178302764893, + "logits/rejected": -1.029137134552002, + "logps/chosen": -691.7874755859375, + "logps/rejected": -813.152099609375, + "loss": 0.3592, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.9073405265808105, + "rewards/margins": 1.246219515800476, + "rewards/rejected": -5.153559684753418, + "step": 347 + }, + { + "epoch": 0.743788405022709, + "grad_norm": 20.32668011498763, + "learning_rate": 1.2973786313255695e-07, + "logits/chosen": -0.8740507960319519, + "logits/rejected": -0.9160584211349487, + "logps/chosen": -662.5859375, + "logps/rejected": -873.8433227539062, + "loss": 0.4097, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.367182731628418, + "rewards/margins": 2.035503625869751, + "rewards/rejected": -6.4026875495910645, + "step": 348 + }, + { + "epoch": 0.7459257280256478, + "grad_norm": 23.02424339442235, + "learning_rate": 1.2770948055523742e-07, + "logits/chosen": -1.0587798357009888, + "logits/rejected": -1.0669668912887573, + "logps/chosen": -809.7626953125, + "logps/rejected": -983.58984375, + "loss": 0.3906, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.2913947105407715, + "rewards/margins": 1.9067097902297974, + "rewards/rejected": -7.1981048583984375, + "step": 349 + }, + { + "epoch": 0.7480630510285867, + "grad_norm": 25.77622479036058, + "learning_rate": 1.2569353510524156e-07, + "logits/chosen": -1.069049596786499, + "logits/rejected": -1.1337969303131104, + "logps/chosen": -720.7489013671875, + "logps/rejected": -938.7985229492188, + "loss": 0.4103, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.965020179748535, + "rewards/margins": 2.0392074584960938, + "rewards/rejected": -7.004227638244629, + "step": 350 + }, + { + "epoch": 0.7502003740315255, + "grad_norm": 17.60740131186996, + "learning_rate": 1.2369013957447015e-07, + "logits/chosen": -1.0574181079864502, + "logits/rejected": -1.0570802688598633, + "logps/chosen": -694.0639038085938, + "logps/rejected": -829.4335327148438, + "loss": 0.3407, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.506197452545166, + "rewards/margins": 1.4661719799041748, + "rewards/rejected": -5.97236967086792, + "step": 351 + }, + { + "epoch": 0.7523376970344643, + "grad_norm": 17.276020504905794, + "learning_rate": 1.216994060526577e-07, + "logits/chosen": -1.1085654497146606, + "logits/rejected": -1.0444343090057373, + "logps/chosen": -699.8790893554688, + "logps/rejected": -857.7581176757812, + "loss": 0.3333, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.61436128616333, + "rewards/margins": 1.533206582069397, + "rewards/rejected": -6.147567272186279, + "step": 352 + }, + { + "epoch": 0.7544750200374032, + "grad_norm": 20.20530099751691, + "learning_rate": 1.197214459211006e-07, + "logits/chosen": -1.028286099433899, + "logits/rejected": -1.0080163478851318, + "logps/chosen": -729.8060913085938, + "logps/rejected": -908.0136108398438, + "loss": 0.3995, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.445433616638184, + "rewards/margins": 1.915571689605713, + "rewards/rejected": -6.361005783081055, + "step": 353 + }, + { + "epoch": 0.756612343040342, + "grad_norm": 21.65892045821199, + "learning_rate": 1.1775636984642612e-07, + "logits/chosen": -1.001996636390686, + "logits/rejected": -0.9989675283432007, + "logps/chosen": -658.6278076171875, + "logps/rejected": -824.7811279296875, + "loss": 0.3989, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.4795989990234375, + "rewards/margins": 1.533904790878296, + "rewards/rejected": -6.0135040283203125, + "step": 354 + }, + { + "epoch": 0.7587496660432808, + "grad_norm": 18.745090346580444, + "learning_rate": 1.1580428777439973e-07, + "logits/chosen": -1.1561853885650635, + "logits/rejected": -1.0848287343978882, + "logps/chosen": -622.0913696289062, + "logps/rejected": -787.5831298828125, + "loss": 0.3734, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.140936851501465, + "rewards/margins": 1.524781346321106, + "rewards/rejected": -5.665718078613281, + "step": 355 + }, + { + "epoch": 0.7608869890462197, + "grad_norm": 18.598176758293135, + "learning_rate": 1.1386530892377399e-07, + "logits/chosen": -0.9880399107933044, + "logits/rejected": -0.9888783097267151, + "logps/chosen": -613.0370483398438, + "logps/rejected": -777.3377075195312, + "loss": 0.3762, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.9182024002075195, + "rewards/margins": 1.6418043375015259, + "rewards/rejected": -5.560007095336914, + "step": 356 + }, + { + "epoch": 0.7630243120491584, + "grad_norm": 20.075874653172413, + "learning_rate": 1.1193954178017814e-07, + "logits/chosen": -0.9616910219192505, + "logits/rejected": -1.0504767894744873, + "logps/chosen": -782.3270874023438, + "logps/rejected": -1025.231689453125, + "loss": 0.4392, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.93320894241333, + "rewards/margins": 2.5418636798858643, + "rewards/rejected": -7.475072383880615, + "step": 357 + }, + { + "epoch": 0.7651616350520972, + "grad_norm": 18.869618371734305, + "learning_rate": 1.1002709409004803e-07, + "logits/chosen": -1.1053738594055176, + "logits/rejected": -1.0864824056625366, + "logps/chosen": -768.8507080078125, + "logps/rejected": -939.0321044921875, + "loss": 0.3226, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.1586713790893555, + "rewards/margins": 1.6820204257965088, + "rewards/rejected": -6.840692043304443, + "step": 358 + }, + { + "epoch": 0.7672989580550361, + "grad_norm": 22.39575999598468, + "learning_rate": 1.0812807285459737e-07, + "logits/chosen": -0.8523386120796204, + "logits/rejected": -0.8684836030006409, + "logps/chosen": -634.4469604492188, + "logps/rejected": -910.783447265625, + "loss": 0.4391, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.161304950714111, + "rewards/margins": 2.574713945388794, + "rewards/rejected": -6.736019134521484, + "step": 359 + }, + { + "epoch": 0.7694362810579749, + "grad_norm": 24.655379460289804, + "learning_rate": 1.0624258432383165e-07, + "logits/chosen": -0.9855338931083679, + "logits/rejected": -0.9286175966262817, + "logps/chosen": -821.5938110351562, + "logps/rejected": -976.9266357421875, + "loss": 0.4425, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.609586715698242, + "rewards/margins": 1.46958327293396, + "rewards/rejected": -7.0791707038879395, + "step": 360 + }, + { + "epoch": 0.7715736040609137, + "grad_norm": 23.07044684161756, + "learning_rate": 1.0437073399060298e-07, + "logits/chosen": -0.9206562638282776, + "logits/rejected": -0.8095078468322754, + "logps/chosen": -618.7409057617188, + "logps/rejected": -815.9017333984375, + "loss": 0.396, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.211564064025879, + "rewards/margins": 1.776960849761963, + "rewards/rejected": -5.988524913787842, + "step": 361 + }, + { + "epoch": 0.7737109270638525, + "grad_norm": 20.255683251830362, + "learning_rate": 1.0251262658470838e-07, + "logits/chosen": -1.0115031003952026, + "logits/rejected": -0.9697583317756653, + "logps/chosen": -671.9834594726562, + "logps/rejected": -856.5125122070312, + "loss": 0.4114, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.316826343536377, + "rewards/margins": 1.731043815612793, + "rewards/rejected": -6.04787015914917, + "step": 362 + }, + { + "epoch": 0.7758482500667914, + "grad_norm": 36.27259699112537, + "learning_rate": 1.0066836606702937e-07, + "logits/chosen": -0.9073708057403564, + "logits/rejected": -1.013731598854065, + "logps/chosen": -687.530029296875, + "logps/rejected": -816.8972778320312, + "loss": 0.5571, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.503976821899414, + "rewards/margins": 1.4232851266860962, + "rewards/rejected": -5.927262306213379, + "step": 363 + }, + { + "epoch": 0.7779855730697302, + "grad_norm": 18.487130214617952, + "learning_rate": 9.883805562371581e-08, + "logits/chosen": -1.096664547920227, + "logits/rejected": -1.151119351387024, + "logps/chosen": -627.2655029296875, + "logps/rejected": -766.6741943359375, + "loss": 0.3511, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.355429649353027, + "rewards/margins": 1.6355644464492798, + "rewards/rejected": -5.990993976593018, + "step": 364 + }, + { + "epoch": 0.7801228960726689, + "grad_norm": 17.467778162939954, + "learning_rate": 9.702179766041299e-08, + "logits/chosen": -1.1129746437072754, + "logits/rejected": -1.0882641077041626, + "logps/chosen": -609.0090942382812, + "logps/rejected": -835.59375, + "loss": 0.3284, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.666825294494629, + "rewards/margins": 2.0798864364624023, + "rewards/rejected": -5.746711730957031, + "step": 365 + }, + { + "epoch": 0.7822602190756078, + "grad_norm": 26.497135911369625, + "learning_rate": 9.521969379653121e-08, + "logits/chosen": -1.0410820245742798, + "logits/rejected": -1.075594425201416, + "logps/chosen": -861.631591796875, + "logps/rejected": -1022.0664672851562, + "loss": 0.4385, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.093762397766113, + "rewards/margins": 1.7978633642196655, + "rewards/rejected": -6.891625881195068, + "step": 366 + }, + { + "epoch": 0.7843975420785466, + "grad_norm": 19.54124775721202, + "learning_rate": 9.343184485956086e-08, + "logits/chosen": -1.0408751964569092, + "logits/rejected": -1.1065208911895752, + "logps/chosen": -597.0819702148438, + "logps/rejected": -755.0318603515625, + "loss": 0.3985, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.7190897464752197, + "rewards/margins": 1.8094736337661743, + "rewards/rejected": -5.528563499450684, + "step": 367 + }, + { + "epoch": 0.7865348650814854, + "grad_norm": 23.844802367349075, + "learning_rate": 9.16583508794308e-08, + "logits/chosen": -0.9044683575630188, + "logits/rejected": -0.8596857786178589, + "logps/chosen": -730.1447143554688, + "logps/rejected": -878.2034912109375, + "loss": 0.3, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.852618217468262, + "rewards/margins": 1.4661897420883179, + "rewards/rejected": -6.318808078765869, + "step": 368 + }, + { + "epoch": 0.7886721880844243, + "grad_norm": 20.562190767658738, + "learning_rate": 8.989931108291197e-08, + "logits/chosen": -0.9048526287078857, + "logits/rejected": -0.9187178611755371, + "logps/chosen": -699.8194580078125, + "logps/rejected": -844.4905395507812, + "loss": 0.3635, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.6599884033203125, + "rewards/margins": 1.4540899991989136, + "rewards/rejected": -6.114078521728516, + "step": 369 + }, + { + "epoch": 0.7908095110873631, + "grad_norm": 20.73718481631679, + "learning_rate": 8.815482388806554e-08, + "logits/chosen": -1.0599019527435303, + "logits/rejected": -1.065873622894287, + "logps/chosen": -760.4301147460938, + "logps/rejected": -946.1746215820312, + "loss": 0.4287, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.120336532592773, + "rewards/margins": 1.838600993156433, + "rewards/rejected": -6.958937168121338, + "step": 370 + }, + { + "epoch": 0.7929468340903019, + "grad_norm": 30.164533991222857, + "learning_rate": 8.642498689873619e-08, + "logits/chosen": -1.0670329332351685, + "logits/rejected": -1.1095917224884033, + "logps/chosen": -768.016357421875, + "logps/rejected": -985.7068481445312, + "loss": 0.4338, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.948676586151123, + "rewards/margins": 2.0501699447631836, + "rewards/rejected": -6.998846530914307, + "step": 371 + }, + { + "epoch": 0.7950841570932408, + "grad_norm": 18.533158040155175, + "learning_rate": 8.47098968990914e-08, + "logits/chosen": -1.212889313697815, + "logits/rejected": -1.174148678779602, + "logps/chosen": -575.9932250976562, + "logps/rejected": -694.8380126953125, + "loss": 0.3543, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.408289670944214, + "rewards/margins": 1.2301357984542847, + "rewards/rejected": -4.638425350189209, + "step": 372 + }, + { + "epoch": 0.7972214800961795, + "grad_norm": 19.740449187608878, + "learning_rate": 8.300964984820672e-08, + "logits/chosen": -0.9624559879302979, + "logits/rejected": -0.9103982448577881, + "logps/chosen": -758.9552001953125, + "logps/rejected": -952.0904541015625, + "loss": 0.385, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.812871932983398, + "rewards/margins": 1.8450303077697754, + "rewards/rejected": -6.657902240753174, + "step": 373 + }, + { + "epoch": 0.7993588030991183, + "grad_norm": 17.857229527882023, + "learning_rate": 8.13243408746961e-08, + "logits/chosen": -1.044665813446045, + "logits/rejected": -1.0194975137710571, + "logps/chosen": -696.4033813476562, + "logps/rejected": -845.8680419921875, + "loss": 0.3787, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.486112594604492, + "rewards/margins": 1.4695513248443604, + "rewards/rejected": -5.955664157867432, + "step": 374 + }, + { + "epoch": 0.8014961261020572, + "grad_norm": 27.402209380453634, + "learning_rate": 7.96540642713902e-08, + "logits/chosen": -1.0042835474014282, + "logits/rejected": -0.9937084317207336, + "logps/chosen": -798.3321533203125, + "logps/rejected": -931.4829711914062, + "loss": 0.4162, + "rewards/accuracies": 0.6875, + "rewards/chosen": -5.05294132232666, + "rewards/margins": 1.302819848060608, + "rewards/rejected": -6.35576057434082, + "step": 375 + }, + { + "epoch": 0.803633449104996, + "grad_norm": 28.3566059208529, + "learning_rate": 7.799891349006019e-08, + "logits/chosen": -1.1246654987335205, + "logits/rejected": -1.1276860237121582, + "logps/chosen": -656.9721069335938, + "logps/rejected": -813.6206665039062, + "loss": 0.4719, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.687532901763916, + "rewards/margins": 1.3982094526290894, + "rewards/rejected": -6.085742473602295, + "step": 376 + }, + { + "epoch": 0.8057707721079348, + "grad_norm": 18.856881189055258, + "learning_rate": 7.635898113618957e-08, + "logits/chosen": -1.2034432888031006, + "logits/rejected": -1.1600438356399536, + "logps/chosen": -585.2130737304688, + "logps/rejected": -785.3953247070312, + "loss": 0.3801, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.6061196327209473, + "rewards/margins": 1.8955157995224, + "rewards/rejected": -5.501636028289795, + "step": 377 + }, + { + "epoch": 0.8079080951108736, + "grad_norm": 21.136594852281714, + "learning_rate": 7.473435896379258e-08, + "logits/chosen": -1.1049622297286987, + "logits/rejected": -1.025770902633667, + "logps/chosen": -697.8673706054688, + "logps/rejected": -841.3628540039062, + "loss": 0.4047, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.117053985595703, + "rewards/margins": 1.336838960647583, + "rewards/rejected": -5.453892707824707, + "step": 378 + }, + { + "epoch": 0.8100454181138125, + "grad_norm": 21.622158153196573, + "learning_rate": 7.312513787028065e-08, + "logits/chosen": -1.018204927444458, + "logits/rejected": -1.0209660530090332, + "logps/chosen": -551.69091796875, + "logps/rejected": -765.5928344726562, + "loss": 0.3705, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.391167402267456, + "rewards/margins": 1.9136013984680176, + "rewards/rejected": -5.304769039154053, + "step": 379 + }, + { + "epoch": 0.8121827411167513, + "grad_norm": 25.692774884542565, + "learning_rate": 7.153140789137667e-08, + "logits/chosen": -1.1156607866287231, + "logits/rejected": -1.1090834140777588, + "logps/chosen": -587.258056640625, + "logps/rejected": -709.8887329101562, + "loss": 0.417, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.669807195663452, + "rewards/margins": 1.1046210527420044, + "rewards/rejected": -4.774428367614746, + "step": 380 + }, + { + "epoch": 0.81432006411969, + "grad_norm": 22.676599376198244, + "learning_rate": 6.995325819607783e-08, + "logits/chosen": -0.8035112619400024, + "logits/rejected": -0.7905966639518738, + "logps/chosen": -724.717529296875, + "logps/rejected": -951.2332153320312, + "loss": 0.3864, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.354774475097656, + "rewards/margins": 2.2559542655944824, + "rewards/rejected": -6.6107282638549805, + "step": 381 + }, + { + "epoch": 0.8164573871226289, + "grad_norm": 20.060527238580573, + "learning_rate": 6.839077708166608e-08, + "logits/chosen": -0.9624785780906677, + "logits/rejected": -0.9216483235359192, + "logps/chosen": -759.6564331054688, + "logps/rejected": -846.4254760742188, + "loss": 0.4242, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.706582546234131, + "rewards/margins": 0.9559379816055298, + "rewards/rejected": -5.662520408630371, + "step": 382 + }, + { + "epoch": 0.8185947101255677, + "grad_norm": 20.450745536438863, + "learning_rate": 6.684405196876843e-08, + "logits/chosen": -0.9839971661567688, + "logits/rejected": -0.9619154930114746, + "logps/chosen": -659.962890625, + "logps/rejected": -834.504638671875, + "loss": 0.4453, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.435713768005371, + "rewards/margins": 1.5213212966918945, + "rewards/rejected": -5.957034587860107, + "step": 383 + }, + { + "epoch": 0.8207320331285065, + "grad_norm": 21.923861116889775, + "learning_rate": 6.531316939646522e-08, + "logits/chosen": -1.0496439933776855, + "logits/rejected": -1.0298651456832886, + "logps/chosen": -652.1452026367188, + "logps/rejected": -761.5989990234375, + "loss": 0.4117, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.186628341674805, + "rewards/margins": 1.098493218421936, + "rewards/rejected": -5.285121917724609, + "step": 384 + }, + { + "epoch": 0.8228693561314454, + "grad_norm": 15.872158871462359, + "learning_rate": 6.379821501744902e-08, + "logits/chosen": -0.9291130900382996, + "logits/rejected": -0.9559027552604675, + "logps/chosen": -637.105712890625, + "logps/rejected": -784.9781494140625, + "loss": 0.3618, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.092860698699951, + "rewards/margins": 1.487864375114441, + "rewards/rejected": -5.580724716186523, + "step": 385 + }, + { + "epoch": 0.8250066791343842, + "grad_norm": 21.170940984738564, + "learning_rate": 6.229927359323156e-08, + "logits/chosen": -1.013567566871643, + "logits/rejected": -0.9818376302719116, + "logps/chosen": -581.1494140625, + "logps/rejected": -750.385009765625, + "loss": 0.3848, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.5493507385253906, + "rewards/margins": 1.4864931106567383, + "rewards/rejected": -5.035844326019287, + "step": 386 + }, + { + "epoch": 0.827144002137323, + "grad_norm": 20.203562100764046, + "learning_rate": 6.081642898940186e-08, + "logits/chosen": -0.9751057624816895, + "logits/rejected": -0.9479618072509766, + "logps/chosen": -535.7874145507812, + "logps/rejected": -717.0790405273438, + "loss": 0.3476, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.4093728065490723, + "rewards/margins": 1.679715871810913, + "rewards/rejected": -5.089088439941406, + "step": 387 + }, + { + "epoch": 0.8292813251402619, + "grad_norm": 16.226862763327308, + "learning_rate": 5.9349764170933674e-08, + "logits/chosen": -0.9501919746398926, + "logits/rejected": -0.9817054867744446, + "logps/chosen": -545.6821899414062, + "logps/rejected": -637.3515625, + "loss": 0.4066, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.451007843017578, + "rewards/margins": 1.0472853183746338, + "rewards/rejected": -4.498293399810791, + "step": 388 + }, + { + "epoch": 0.8314186481432007, + "grad_norm": 21.301359722964037, + "learning_rate": 5.789936119754407e-08, + "logits/chosen": -1.070288062095642, + "logits/rejected": -1.03058660030365, + "logps/chosen": -773.905517578125, + "logps/rejected": -884.7736206054688, + "loss": 0.3989, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.410757064819336, + "rewards/margins": 1.2033069133758545, + "rewards/rejected": -5.6140642166137695, + "step": 389 + }, + { + "epoch": 0.8335559711461394, + "grad_norm": 16.155956822127486, + "learning_rate": 5.646530121910162e-08, + "logits/chosen": -0.9560039043426514, + "logits/rejected": -0.9677013158798218, + "logps/chosen": -555.0410766601562, + "logps/rejected": -647.4817504882812, + "loss": 0.3702, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.367013931274414, + "rewards/margins": 1.1322979927062988, + "rewards/rejected": -4.499312400817871, + "step": 390 + }, + { + "epoch": 0.8356932941490782, + "grad_norm": 17.445283127579135, + "learning_rate": 5.5047664471086284e-08, + "logits/chosen": -1.0367249250411987, + "logits/rejected": -1.0191857814788818, + "logps/chosen": -623.27978515625, + "logps/rejected": -845.25048828125, + "loss": 0.354, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.8465986251831055, + "rewards/margins": 2.2664730548858643, + "rewards/rejected": -6.113072395324707, + "step": 391 + }, + { + "epoch": 0.8378306171520171, + "grad_norm": 17.389408198261865, + "learning_rate": 5.3646530270100556e-08, + "logits/chosen": -1.1824917793273926, + "logits/rejected": -1.1786624193191528, + "logps/chosen": -629.4869384765625, + "logps/rejected": -910.8460693359375, + "loss": 0.3307, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.6656405925750732, + "rewards/margins": 2.664851188659668, + "rewards/rejected": -6.33049201965332, + "step": 392 + }, + { + "epoch": 0.8399679401549559, + "grad_norm": 16.107124952267284, + "learning_rate": 5.2261977009431193e-08, + "logits/chosen": -1.2528231143951416, + "logits/rejected": -1.2414242029190063, + "logps/chosen": -679.169189453125, + "logps/rejected": -904.231689453125, + "loss": 0.3437, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.949631690979004, + "rewards/margins": 1.9819270372390747, + "rewards/rejected": -5.931559085845947, + "step": 393 + }, + { + "epoch": 0.8421052631578947, + "grad_norm": 14.65787560767015, + "learning_rate": 5.089408215466349e-08, + "logits/chosen": -1.1920839548110962, + "logits/rejected": -1.1330435276031494, + "logps/chosen": -591.995849609375, + "logps/rejected": -754.32421875, + "loss": 0.3777, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.461061954498291, + "rewards/margins": 1.4271600246429443, + "rewards/rejected": -4.8882222175598145, + "step": 394 + }, + { + "epoch": 0.8442425861608336, + "grad_norm": 19.973781830012914, + "learning_rate": 4.9542922239346865e-08, + "logits/chosen": -0.9532175660133362, + "logits/rejected": -0.9305418729782104, + "logps/chosen": -700.0613403320312, + "logps/rejected": -854.81201171875, + "loss": 0.4104, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.186321258544922, + "rewards/margins": 1.5199509859085083, + "rewards/rejected": -5.706272602081299, + "step": 395 + }, + { + "epoch": 0.8463799091637724, + "grad_norm": 26.08541321380337, + "learning_rate": 4.8208572860713086e-08, + "logits/chosen": -1.149269461631775, + "logits/rejected": -1.1293056011199951, + "logps/chosen": -618.3247680664062, + "logps/rejected": -777.4251708984375, + "loss": 0.4851, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.4766008853912354, + "rewards/margins": 1.5825449228286743, + "rewards/rejected": -5.059146404266357, + "step": 396 + }, + { + "epoch": 0.8485172321667112, + "grad_norm": 29.339785563765055, + "learning_rate": 4.689110867544645e-08, + "logits/chosen": -1.015437126159668, + "logits/rejected": -0.9925183057785034, + "logps/chosen": -679.9268188476562, + "logps/rejected": -870.37890625, + "loss": 0.344, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.385658264160156, + "rewards/margins": 1.9707388877868652, + "rewards/rejected": -6.35639762878418, + "step": 397 + }, + { + "epoch": 0.85065455516965, + "grad_norm": 19.625508223329977, + "learning_rate": 4.5590603395506626e-08, + "logits/chosen": -1.0667216777801514, + "logits/rejected": -1.049599289894104, + "logps/chosen": -589.4276123046875, + "logps/rejected": -712.3489379882812, + "loss": 0.4014, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.779334306716919, + "rewards/margins": 1.0566929578781128, + "rewards/rejected": -4.836027145385742, + "step": 398 + }, + { + "epoch": 0.8527918781725888, + "grad_norm": 19.959692153012657, + "learning_rate": 4.430712978400462e-08, + "logits/chosen": -1.0223028659820557, + "logits/rejected": -1.0172353982925415, + "logps/chosen": -494.1994323730469, + "logps/rejected": -618.9550170898438, + "loss": 0.4284, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.038055658340454, + "rewards/margins": 1.1104681491851807, + "rewards/rejected": -4.148523807525635, + "step": 399 + }, + { + "epoch": 0.8549292011755276, + "grad_norm": 22.852143621544176, + "learning_rate": 4.304075965113171e-08, + "logits/chosen": -1.2095826864242554, + "logits/rejected": -1.1936359405517578, + "logps/chosen": -572.5497436523438, + "logps/rejected": -702.556396484375, + "loss": 0.3827, + "rewards/accuracies": 0.8125, + "rewards/chosen": -3.307962656021118, + "rewards/margins": 1.31619131565094, + "rewards/rejected": -4.624154090881348, + "step": 400 + }, + { + "epoch": 0.8549292011755276, + "eval_logits/chosen": -1.128338098526001, + "eval_logits/rejected": -1.1495376825332642, + "eval_logps/chosen": -710.064697265625, + "eval_logps/rejected": -863.7308349609375, + "eval_loss": 0.3871653079986572, + "eval_rewards/accuracies": 0.8709677457809448, + "eval_rewards/chosen": -4.3693461418151855, + "eval_rewards/margins": 1.5976247787475586, + "eval_rewards/rejected": -5.966970920562744, + "eval_runtime": 127.1231, + "eval_samples_per_second": 15.426, + "eval_steps_per_second": 0.488, + "step": 400 + }, + { + "epoch": 0.8570665241784665, + "grad_norm": 20.918890009764137, + "learning_rate": 4.1791563850141695e-08, + "logits/chosen": -0.9988707900047302, + "logits/rejected": -1.0228999853134155, + "logps/chosen": -705.0687255859375, + "logps/rejected": -993.6966552734375, + "loss": 0.3109, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.109606742858887, + "rewards/margins": 2.890212297439575, + "rewards/rejected": -6.999818801879883, + "step": 401 + }, + { + "epoch": 0.8592038471814053, + "grad_norm": 22.50167745931539, + "learning_rate": 4.0559612273386616e-08, + "logits/chosen": -1.098199725151062, + "logits/rejected": -1.1250154972076416, + "logps/chosen": -785.855224609375, + "logps/rejected": -917.9451904296875, + "loss": 0.3891, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.712646484375, + "rewards/margins": 1.341720700263977, + "rewards/rejected": -6.0543670654296875, + "step": 402 + }, + { + "epoch": 0.8613411701843441, + "grad_norm": 16.355069952426618, + "learning_rate": 3.934497384840607e-08, + "logits/chosen": -0.9095240235328674, + "logits/rejected": -0.8507342338562012, + "logps/chosen": -621.5723876953125, + "logps/rejected": -791.5116577148438, + "loss": 0.4111, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.8870487213134766, + "rewards/margins": 1.6334927082061768, + "rewards/rejected": -5.520541667938232, + "step": 403 + }, + { + "epoch": 0.863478493187283, + "grad_norm": 20.03294448892495, + "learning_rate": 3.8147716534071274e-08, + "logits/chosen": -1.0887154340744019, + "logits/rejected": -1.095017671585083, + "logps/chosen": -605.3096923828125, + "logps/rejected": -669.5009155273438, + "loss": 0.4435, + "rewards/accuracies": 0.5625, + "rewards/chosen": -3.492586374282837, + "rewards/margins": 0.6767375469207764, + "rewards/rejected": -4.169323921203613, + "step": 404 + }, + { + "epoch": 0.8656158161902218, + "grad_norm": 19.638131692176458, + "learning_rate": 3.696790731678217e-08, + "logits/chosen": -1.0262949466705322, + "logits/rejected": -1.0150538682937622, + "logps/chosen": -797.56689453125, + "logps/rejected": -929.2540893554688, + "loss": 0.4095, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.939427852630615, + "rewards/margins": 1.520028829574585, + "rewards/rejected": -6.459456920623779, + "step": 405 + }, + { + "epoch": 0.8677531391931605, + "grad_norm": 19.64316736105873, + "learning_rate": 3.5805612206719885e-08, + "logits/chosen": -0.9092820882797241, + "logits/rejected": -0.9483243227005005, + "logps/chosen": -681.32958984375, + "logps/rejected": -848.2050170898438, + "loss": 0.349, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.447098255157471, + "rewards/margins": 1.769409418106079, + "rewards/rejected": -6.216507911682129, + "step": 406 + }, + { + "epoch": 0.8698904621960993, + "grad_norm": 16.2039010339986, + "learning_rate": 3.466089623415333e-08, + "logits/chosen": -0.9941847324371338, + "logits/rejected": -0.9800211787223816, + "logps/chosen": -583.2901000976562, + "logps/rejected": -831.3629150390625, + "loss": 0.3707, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.81510329246521, + "rewards/margins": 2.43589448928833, + "rewards/rejected": -6.250997543334961, + "step": 407 + }, + { + "epoch": 0.8720277851990382, + "grad_norm": 17.593791491624575, + "learning_rate": 3.3533823445800965e-08, + "logits/chosen": -1.0939316749572754, + "logits/rejected": -1.0538439750671387, + "logps/chosen": -563.0457763671875, + "logps/rejected": -702.2050170898438, + "loss": 0.3908, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.7340145111083984, + "rewards/margins": 1.5319364070892334, + "rewards/rejected": -5.2659502029418945, + "step": 408 + }, + { + "epoch": 0.874165108201977, + "grad_norm": 17.753144065442136, + "learning_rate": 3.242445690124729e-08, + "logits/chosen": -1.0609853267669678, + "logits/rejected": -1.0725332498550415, + "logps/chosen": -745.48046875, + "logps/rejected": -863.9573974609375, + "loss": 0.3536, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.6711931228637695, + "rewards/margins": 1.1300690174102783, + "rewards/rejected": -5.801262378692627, + "step": 409 + }, + { + "epoch": 0.8763024312049158, + "grad_norm": 22.5406397148522, + "learning_rate": 3.133285866941445e-08, + "logits/chosen": -0.9890793561935425, + "logits/rejected": -1.017478585243225, + "logps/chosen": -489.8434143066406, + "logps/rejected": -649.6760864257812, + "loss": 0.463, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.189424991607666, + "rewards/margins": 1.429877758026123, + "rewards/rejected": -4.619302749633789, + "step": 410 + }, + { + "epoch": 0.8784397542078547, + "grad_norm": 16.06885831310049, + "learning_rate": 3.025908982508966e-08, + "logits/chosen": -1.04689359664917, + "logits/rejected": -0.9806681871414185, + "logps/chosen": -767.8428344726562, + "logps/rejected": -1035.4447021484375, + "loss": 0.3126, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.467133045196533, + "rewards/margins": 2.5061354637145996, + "rewards/rejected": -6.973268985748291, + "step": 411 + }, + { + "epoch": 0.8805770772107935, + "grad_norm": 19.662285734659836, + "learning_rate": 2.9203210445508332e-08, + "logits/chosen": -1.0549553632736206, + "logits/rejected": -1.008999228477478, + "logps/chosen": -739.88720703125, + "logps/rejected": -918.3170166015625, + "loss": 0.4546, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.464229583740234, + "rewards/margins": 1.5558303594589233, + "rewards/rejected": -6.0200605392456055, + "step": 412 + }, + { + "epoch": 0.8827144002137323, + "grad_norm": 27.990757995621482, + "learning_rate": 2.8165279606992298e-08, + "logits/chosen": -0.9684846997261047, + "logits/rejected": -0.9231318235397339, + "logps/chosen": -719.86083984375, + "logps/rejected": -912.4309692382812, + "loss": 0.3979, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.8175811767578125, + "rewards/margins": 1.912165641784668, + "rewards/rejected": -6.729746341705322, + "step": 413 + }, + { + "epoch": 0.8848517232166712, + "grad_norm": 17.729091244130924, + "learning_rate": 2.7145355381644793e-08, + "logits/chosen": -1.0500560998916626, + "logits/rejected": -1.0563499927520752, + "logps/chosen": -557.0240478515625, + "logps/rejected": -717.523681640625, + "loss": 0.3655, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.6325652599334717, + "rewards/margins": 1.5925527811050415, + "rewards/rejected": -5.225118160247803, + "step": 414 + }, + { + "epoch": 0.88698904621961, + "grad_norm": 19.8351911008517, + "learning_rate": 2.6143494834101153e-08, + "logits/chosen": -0.9061897993087769, + "logits/rejected": -0.9052097797393799, + "logps/chosen": -704.2842407226562, + "logps/rejected": -743.07958984375, + "loss": 0.3664, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.4450836181640625, + "rewards/margins": 0.44577062129974365, + "rewards/rejected": -4.890854835510254, + "step": 415 + }, + { + "epoch": 0.8891263692225487, + "grad_norm": 37.85808985299137, + "learning_rate": 2.5159754018336403e-08, + "logits/chosen": -1.1449795961380005, + "logits/rejected": -1.1638754606246948, + "logps/chosen": -737.4818115234375, + "logps/rejected": -989.4339599609375, + "loss": 0.4523, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.679790019989014, + "rewards/margins": 2.3331949710845947, + "rewards/rejected": -7.012984275817871, + "step": 416 + }, + { + "epoch": 0.8912636922254876, + "grad_norm": 21.579885808296115, + "learning_rate": 2.4194187974528553e-08, + "logits/chosen": -1.104061484336853, + "logits/rejected": -1.0894217491149902, + "logps/chosen": -679.447509765625, + "logps/rejected": -833.4044189453125, + "loss": 0.3835, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.530219554901123, + "rewards/margins": 1.5757548809051514, + "rewards/rejected": -6.105974197387695, + "step": 417 + }, + { + "epoch": 0.8934010152284264, + "grad_norm": 18.32956314016428, + "learning_rate": 2.3246850725979427e-08, + "logits/chosen": -1.0385122299194336, + "logits/rejected": -1.0385818481445312, + "logps/chosen": -666.721435546875, + "logps/rejected": -787.4942626953125, + "loss": 0.3856, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.093487739562988, + "rewards/margins": 1.256799340248108, + "rewards/rejected": -5.350286960601807, + "step": 418 + }, + { + "epoch": 0.8955383382313652, + "grad_norm": 26.323713896853675, + "learning_rate": 2.2317795276091977e-08, + "logits/chosen": -0.9605960249900818, + "logits/rejected": -0.9666808247566223, + "logps/chosen": -626.7743530273438, + "logps/rejected": -742.9835205078125, + "loss": 0.4861, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.809014320373535, + "rewards/margins": 1.3489842414855957, + "rewards/rejected": -5.157999038696289, + "step": 419 + }, + { + "epoch": 0.897675661234304, + "grad_norm": 29.92038686319786, + "learning_rate": 2.1407073605404862e-08, + "logits/chosen": -1.0402368307113647, + "logits/rejected": -0.9794477224349976, + "logps/chosen": -743.0745849609375, + "logps/rejected": -964.0947875976562, + "loss": 0.4477, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.6502366065979, + "rewards/margins": 2.061614990234375, + "rewards/rejected": -6.711851119995117, + "step": 420 + }, + { + "epoch": 0.8998129842372429, + "grad_norm": 18.598312258529717, + "learning_rate": 2.0514736668683917e-08, + "logits/chosen": -1.1911952495574951, + "logits/rejected": -1.2088751792907715, + "logps/chosen": -688.5134887695312, + "logps/rejected": -876.8580932617188, + "loss": 0.4087, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.259193420410156, + "rewards/margins": 1.567337155342102, + "rewards/rejected": -5.826530456542969, + "step": 421 + }, + { + "epoch": 0.9019503072401817, + "grad_norm": 23.213199916317794, + "learning_rate": 1.964083439207135e-08, + "logits/chosen": -1.1022056341171265, + "logits/rejected": -1.1523065567016602, + "logps/chosen": -729.16259765625, + "logps/rejected": -873.5059204101562, + "loss": 0.4305, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.697969913482666, + "rewards/margins": 1.660890817642212, + "rewards/rejected": -6.358860969543457, + "step": 422 + }, + { + "epoch": 0.9040876302431204, + "grad_norm": 17.78006990070099, + "learning_rate": 1.878541567029251e-08, + "logits/chosen": -1.0770941972732544, + "logits/rejected": -1.0126092433929443, + "logps/chosen": -606.9859008789062, + "logps/rejected": -791.86474609375, + "loss": 0.3209, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.224216461181641, + "rewards/margins": 1.6672827005386353, + "rewards/rejected": -5.891498565673828, + "step": 423 + }, + { + "epoch": 0.9062249532460593, + "grad_norm": 28.8620946540648, + "learning_rate": 1.794852836392009e-08, + "logits/chosen": -0.9167647957801819, + "logits/rejected": -0.9531383514404297, + "logps/chosen": -565.0890502929688, + "logps/rejected": -742.736328125, + "loss": 0.4255, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.4048359394073486, + "rewards/margins": 1.6988798379898071, + "rewards/rejected": -5.103715896606445, + "step": 424 + }, + { + "epoch": 0.9083622762489981, + "grad_norm": 20.499467136576563, + "learning_rate": 1.7130219296696264e-08, + "logits/chosen": -0.9955915212631226, + "logits/rejected": -1.0456228256225586, + "logps/chosen": -657.0418090820312, + "logps/rejected": -762.1922607421875, + "loss": 0.3629, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.7994446754455566, + "rewards/margins": 1.2025730609893799, + "rewards/rejected": -5.002017974853516, + "step": 425 + }, + { + "epoch": 0.9104995992519369, + "grad_norm": 17.206845074940055, + "learning_rate": 1.6330534252913086e-08, + "logits/chosen": -1.0955368280410767, + "logits/rejected": -1.081055998802185, + "logps/chosen": -706.4891967773438, + "logps/rejected": -900.5104370117188, + "loss": 0.3426, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.797817230224609, + "rewards/margins": 1.9311976432800293, + "rewards/rejected": -6.729015350341797, + "step": 426 + }, + { + "epoch": 0.9126369222548758, + "grad_norm": 17.119036708717413, + "learning_rate": 1.5549517974850724e-08, + "logits/chosen": -0.8513895273208618, + "logits/rejected": -0.8118205070495605, + "logps/chosen": -611.9838256835938, + "logps/rejected": -755.0986328125, + "loss": 0.3786, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.1998372077941895, + "rewards/margins": 1.362562656402588, + "rewards/rejected": -5.562399864196777, + "step": 427 + }, + { + "epoch": 0.9147742452578146, + "grad_norm": 30.666142274697034, + "learning_rate": 1.4787214160274341e-08, + "logits/chosen": -1.1465550661087036, + "logits/rejected": -1.1435879468917847, + "logps/chosen": -637.793701171875, + "logps/rejected": -803.086669921875, + "loss": 0.4319, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.044920444488525, + "rewards/margins": 1.6718565225601196, + "rewards/rejected": -5.7167768478393555, + "step": 428 + }, + { + "epoch": 0.9169115682607534, + "grad_norm": 17.479325335479295, + "learning_rate": 1.4043665459988868e-08, + "logits/chosen": -1.1191812753677368, + "logits/rejected": -1.0620485544204712, + "logps/chosen": -617.3558349609375, + "logps/rejected": -858.5902099609375, + "loss": 0.3978, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.054196357727051, + "rewards/margins": 2.3324999809265137, + "rewards/rejected": -6.3866963386535645, + "step": 429 + }, + { + "epoch": 0.9190488912636923, + "grad_norm": 19.870927489532274, + "learning_rate": 1.3318913475452925e-08, + "logits/chosen": -1.1520562171936035, + "logits/rejected": -1.0596178770065308, + "logps/chosen": -827.845947265625, + "logps/rejected": -1057.0477294921875, + "loss": 0.3633, + "rewards/accuracies": 0.8125, + "rewards/chosen": -5.0968828201293945, + "rewards/margins": 2.2637462615966797, + "rewards/rejected": -7.360629081726074, + "step": 430 + }, + { + "epoch": 0.921186214266631, + "grad_norm": 19.001198221145234, + "learning_rate": 1.2612998756451366e-08, + "logits/chosen": -1.0448460578918457, + "logits/rejected": -1.0036524534225464, + "logps/chosen": -565.2469482421875, + "logps/rejected": -768.9799194335938, + "loss": 0.3764, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.4013609886169434, + "rewards/margins": 2.0993523597717285, + "rewards/rejected": -5.500713348388672, + "step": 431 + }, + { + "epoch": 0.9233235372695698, + "grad_norm": 24.185158250695462, + "learning_rate": 1.192596079882613e-08, + "logits/chosen": -1.016557216644287, + "logits/rejected": -0.9917696714401245, + "logps/chosen": -754.6041259765625, + "logps/rejected": -1039.5848388671875, + "loss": 0.4467, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.575927257537842, + "rewards/margins": 2.5993194580078125, + "rewards/rejected": -7.175246715545654, + "step": 432 + }, + { + "epoch": 0.9254608602725087, + "grad_norm": 18.630615195275038, + "learning_rate": 1.1257838042266754e-08, + "logits/chosen": -0.9740123152732849, + "logits/rejected": -0.9422295689582825, + "logps/chosen": -684.7318115234375, + "logps/rejected": -853.54931640625, + "loss": 0.3922, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.472279071807861, + "rewards/margins": 1.5442816019058228, + "rewards/rejected": -6.016561031341553, + "step": 433 + }, + { + "epoch": 0.9275981832754475, + "grad_norm": 19.666494529888574, + "learning_rate": 1.0608667868159592e-08, + "logits/chosen": -1.0905375480651855, + "logits/rejected": -1.0876115560531616, + "logps/chosen": -828.9867553710938, + "logps/rejected": -1100.01318359375, + "loss": 0.3426, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.9841694831848145, + "rewards/margins": 2.305903911590576, + "rewards/rejected": -7.290073394775391, + "step": 434 + }, + { + "epoch": 0.9297355062783863, + "grad_norm": 18.138692746396494, + "learning_rate": 9.978486597496372e-09, + "logits/chosen": -0.9997820854187012, + "logits/rejected": -0.997348427772522, + "logps/chosen": -644.03955078125, + "logps/rejected": -883.5596923828125, + "loss": 0.3441, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.433194160461426, + "rewards/margins": 2.0547845363616943, + "rewards/rejected": -6.487978935241699, + "step": 435 + }, + { + "epoch": 0.9318728292813251, + "grad_norm": 22.806909647633304, + "learning_rate": 9.367329488842001e-09, + "logits/chosen": -1.0480362176895142, + "logits/rejected": -1.025497317314148, + "logps/chosen": -719.160400390625, + "logps/rejected": -870.7994384765625, + "loss": 0.3881, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.514778137207031, + "rewards/margins": 1.344374656677246, + "rewards/rejected": -5.859152317047119, + "step": 436 + }, + { + "epoch": 0.934010152284264, + "grad_norm": 34.30034026086993, + "learning_rate": 8.775230736361733e-09, + "logits/chosen": -0.9019148349761963, + "logits/rejected": -1.021255612373352, + "logps/chosen": -610.4000244140625, + "logps/rejected": -666.5303344726562, + "loss": 0.5228, + "rewards/accuracies": 0.625, + "rewards/chosen": -3.8257713317871094, + "rewards/margins": 0.6458057165145874, + "rewards/rejected": -4.471577167510986, + "step": 437 + }, + { + "epoch": 0.9361474752872028, + "grad_norm": 26.008895134527737, + "learning_rate": 8.202223467908299e-09, + "logits/chosen": -1.208720088005066, + "logits/rejected": -1.1576639413833618, + "logps/chosen": -704.7356567382812, + "logps/rejected": -935.2647094726562, + "loss": 0.429, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.226789474487305, + "rewards/margins": 2.07527494430542, + "rewards/rejected": -6.302064418792725, + "step": 438 + }, + { + "epoch": 0.9382847982901416, + "grad_norm": 23.1664224383978, + "learning_rate": 7.64833974316801e-09, + "logits/chosen": -0.9366050958633423, + "logits/rejected": -1.0036077499389648, + "logps/chosen": -692.1240844726562, + "logps/rejected": -962.029541015625, + "loss": 0.3745, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.578640937805176, + "rewards/margins": 2.751880407333374, + "rewards/rejected": -7.330521583557129, + "step": 439 + }, + { + "epoch": 0.9404221212930804, + "grad_norm": 25.822513994850507, + "learning_rate": 7.113610551867583e-09, + "logits/chosen": -1.1607967615127563, + "logits/rejected": -1.162611484527588, + "logps/chosen": -727.8070678710938, + "logps/rejected": -895.8046875, + "loss": 0.4345, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.304068565368652, + "rewards/margins": 1.6857025623321533, + "rewards/rejected": -5.989770889282227, + "step": 440 + }, + { + "epoch": 0.9425594442960192, + "grad_norm": 24.174938576336224, + "learning_rate": 6.5980658120396106e-09, + "logits/chosen": -1.013208270072937, + "logits/rejected": -0.9189801216125488, + "logps/chosen": -747.7554321289062, + "logps/rejected": -1023.7532348632812, + "loss": 0.4307, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.824457168579102, + "rewards/margins": 2.6137688159942627, + "rewards/rejected": -7.438226222991943, + "step": 441 + }, + { + "epoch": 0.944696767298958, + "grad_norm": 21.66924362068404, + "learning_rate": 6.101734368349104e-09, + "logits/chosen": -1.089997410774231, + "logits/rejected": -1.1180981397628784, + "logps/chosen": -580.3588256835938, + "logps/rejected": -653.803955078125, + "loss": 0.3454, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.4357962608337402, + "rewards/margins": 0.8316761255264282, + "rewards/rejected": -4.267472267150879, + "step": 442 + }, + { + "epoch": 0.9468340903018969, + "grad_norm": 18.92185198276508, + "learning_rate": 5.624643990479616e-09, + "logits/chosen": -0.9731349349021912, + "logits/rejected": -1.0123114585876465, + "logps/chosen": -744.20068359375, + "logps/rejected": -880.1783447265625, + "loss": 0.4027, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.492010116577148, + "rewards/margins": 1.5610084533691406, + "rewards/rejected": -6.053019046783447, + "step": 443 + }, + { + "epoch": 0.9489714133048357, + "grad_norm": 19.40472791239129, + "learning_rate": 5.166821371579277e-09, + "logits/chosen": -0.8690105676651001, + "logits/rejected": -0.9122138023376465, + "logps/chosen": -592.9273071289062, + "logps/rejected": -692.8623046875, + "loss": 0.3917, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.9664011001586914, + "rewards/margins": 1.128931999206543, + "rewards/rejected": -5.095333099365234, + "step": 444 + }, + { + "epoch": 0.9511087363077745, + "grad_norm": 17.606366912659396, + "learning_rate": 4.7282921267675306e-09, + "logits/chosen": -1.1353975534439087, + "logits/rejected": -1.069037914276123, + "logps/chosen": -617.16845703125, + "logps/rejected": -833.6134033203125, + "loss": 0.3496, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.9898786544799805, + "rewards/margins": 1.8373284339904785, + "rewards/rejected": -5.827206611633301, + "step": 445 + }, + { + "epoch": 0.9532460593107134, + "grad_norm": 22.334810036861253, + "learning_rate": 4.309080791701819e-09, + "logits/chosen": -0.9705511927604675, + "logits/rejected": -0.9864068031311035, + "logps/chosen": -772.231689453125, + "logps/rejected": -924.7553100585938, + "loss": 0.3883, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.6818671226501465, + "rewards/margins": 1.767922043800354, + "rewards/rejected": -6.449788570404053, + "step": 446 + }, + { + "epoch": 0.9553833823136522, + "grad_norm": 18.928358142495153, + "learning_rate": 3.909210821205016e-09, + "logits/chosen": -1.0940232276916504, + "logits/rejected": -1.1222940683364868, + "logps/chosen": -569.94482421875, + "logps/rejected": -690.92578125, + "loss": 0.3731, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.7058024406433105, + "rewards/margins": 1.2121186256408691, + "rewards/rejected": -4.91792106628418, + "step": 447 + }, + { + "epoch": 0.957520705316591, + "grad_norm": 19.387388769733043, + "learning_rate": 3.528704587952963e-09, + "logits/chosen": -0.9721835255622864, + "logits/rejected": -0.9633288383483887, + "logps/chosen": -691.5062255859375, + "logps/rejected": -864.804443359375, + "loss": 0.3357, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.21799898147583, + "rewards/margins": 1.6493499279022217, + "rewards/rejected": -5.867349147796631, + "step": 448 + }, + { + "epoch": 0.9596580283195298, + "grad_norm": 15.631110311383365, + "learning_rate": 3.1675833812228547e-09, + "logits/chosen": -1.143801212310791, + "logits/rejected": -1.1312353610992432, + "logps/chosen": -591.1502685546875, + "logps/rejected": -746.0208129882812, + "loss": 0.4032, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.8137049674987793, + "rewards/margins": 1.4380216598510742, + "rewards/rejected": -5.2517266273498535, + "step": 449 + }, + { + "epoch": 0.9617953513224686, + "grad_norm": 23.63954938708458, + "learning_rate": 2.825867405701826e-09, + "logits/chosen": -1.0871686935424805, + "logits/rejected": -1.0616105794906616, + "logps/chosen": -785.9561157226562, + "logps/rejected": -949.911376953125, + "loss": 0.4622, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.555358409881592, + "rewards/margins": 1.6683781147003174, + "rewards/rejected": -6.22373628616333, + "step": 450 + }, + { + "epoch": 0.9639326743254074, + "grad_norm": 22.04528865685746, + "learning_rate": 2.503575780356926e-09, + "logits/chosen": -1.1157352924346924, + "logits/rejected": -1.122366189956665, + "logps/chosen": -687.6873779296875, + "logps/rejected": -789.6235961914062, + "loss": 0.4453, + "rewards/accuracies": 0.6875, + "rewards/chosen": -4.202605724334717, + "rewards/margins": 1.0868072509765625, + "rewards/rejected": -5.289412975311279, + "step": 451 + }, + { + "epoch": 0.9660699973283462, + "grad_norm": 20.620417690774662, + "learning_rate": 2.2007265373650885e-09, + "logits/chosen": -1.181131362915039, + "logits/rejected": -1.1398506164550781, + "logps/chosen": -752.2744750976562, + "logps/rejected": -903.8162231445312, + "loss": 0.4466, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.831580638885498, + "rewards/margins": 1.5455623865127563, + "rewards/rejected": -6.377142906188965, + "step": 452 + }, + { + "epoch": 0.9682073203312851, + "grad_norm": 21.990068975057824, + "learning_rate": 1.9173366211043483e-09, + "logits/chosen": -1.066049337387085, + "logits/rejected": -1.145350694656372, + "logps/chosen": -732.5936889648438, + "logps/rejected": -820.995361328125, + "loss": 0.4411, + "rewards/accuracies": 0.625, + "rewards/chosen": -4.793266296386719, + "rewards/margins": 0.9948696494102478, + "rewards/rejected": -5.788136005401611, + "step": 453 + }, + { + "epoch": 0.9703446433342239, + "grad_norm": 18.52277369573019, + "learning_rate": 1.653421887205747e-09, + "logits/chosen": -1.002540111541748, + "logits/rejected": -1.0223251581192017, + "logps/chosen": -650.927978515625, + "logps/rejected": -728.2699584960938, + "loss": 0.3655, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.248347759246826, + "rewards/margins": 0.7103351354598999, + "rewards/rejected": -4.958683013916016, + "step": 454 + }, + { + "epoch": 0.9724819663371627, + "grad_norm": 33.05665306914434, + "learning_rate": 1.408997101666326e-09, + "logits/chosen": -1.1706193685531616, + "logits/rejected": -1.108201265335083, + "logps/chosen": -693.7764282226562, + "logps/rejected": -911.11962890625, + "loss": 0.4464, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.211030006408691, + "rewards/margins": 1.9548094272613525, + "rewards/rejected": -6.165839195251465, + "step": 455 + }, + { + "epoch": 0.9746192893401016, + "grad_norm": 18.73958161040263, + "learning_rate": 1.1840759400229328e-09, + "logits/chosen": -0.9501774907112122, + "logits/rejected": -0.9423767924308777, + "logps/chosen": -872.7874755859375, + "logps/rejected": -1080.9110107421875, + "loss": 0.4383, + "rewards/accuracies": 0.875, + "rewards/chosen": -5.329258441925049, + "rewards/margins": 1.8498148918151855, + "rewards/rejected": -7.179073810577393, + "step": 456 + }, + { + "epoch": 0.9767566123430403, + "grad_norm": 23.633155328899434, + "learning_rate": 9.786709865869547e-10, + "logits/chosen": -1.0224264860153198, + "logits/rejected": -0.9899497032165527, + "logps/chosen": -691.5697631835938, + "logps/rejected": -1032.654052734375, + "loss": 0.3647, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.561187744140625, + "rewards/margins": 3.524183750152588, + "rewards/rejected": -8.085371971130371, + "step": 457 + }, + { + "epoch": 0.9788939353459791, + "grad_norm": 20.02815998548278, + "learning_rate": 7.927937337403711e-10, + "logits/chosen": -1.1305997371673584, + "logits/rejected": -1.1589406728744507, + "logps/chosen": -720.5179443359375, + "logps/rejected": -914.910400390625, + "loss": 0.413, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.451896667480469, + "rewards/margins": 2.01759672164917, + "rewards/rejected": -6.4694929122924805, + "step": 458 + }, + { + "epoch": 0.981031258348918, + "grad_norm": 21.2779508066291, + "learning_rate": 6.264545812926958e-10, + "logits/chosen": -1.0647788047790527, + "logits/rejected": -1.044015884399414, + "logps/chosen": -644.20263671875, + "logps/rejected": -890.7341918945312, + "loss": 0.4476, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.727494239807129, + "rewards/margins": 2.3570847511291504, + "rewards/rejected": -6.0845794677734375, + "step": 459 + }, + { + "epoch": 0.9831685813518568, + "grad_norm": 19.021711845408657, + "learning_rate": 4.796628358991583e-10, + "logits/chosen": -1.0134469270706177, + "logits/rejected": -1.0272549390792847, + "logps/chosen": -688.755859375, + "logps/rejected": -888.3029174804688, + "loss": 0.3752, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.3303303718566895, + "rewards/margins": 1.7046260833740234, + "rewards/rejected": -6.034955978393555, + "step": 460 + }, + { + "epoch": 0.9853059043547956, + "grad_norm": 22.296101394440072, + "learning_rate": 3.524267105398937e-10, + "logits/chosen": -1.0911214351654053, + "logits/rejected": -1.1538598537445068, + "logps/chosen": -693.211181640625, + "logps/rejected": -855.1487426757812, + "loss": 0.3781, + "rewards/accuracies": 0.875, + "rewards/chosen": -4.124536514282227, + "rewards/margins": 1.6871302127838135, + "rewards/rejected": -5.811666965484619, + "step": 461 + }, + { + "epoch": 0.9874432273577345, + "grad_norm": 18.18277828641047, + "learning_rate": 2.447533240604871e-10, + "logits/chosen": -1.0473253726959229, + "logits/rejected": -1.0253962278366089, + "logps/chosen": -531.8089599609375, + "logps/rejected": -790.6080322265625, + "loss": 0.3867, + "rewards/accuracies": 0.9375, + "rewards/chosen": -3.4056613445281982, + "rewards/margins": 2.266097068786621, + "rewards/rejected": -5.671758651733398, + "step": 462 + }, + { + "epoch": 0.9895805503606733, + "grad_norm": 20.702683706379894, + "learning_rate": 1.5664870077383707e-10, + "logits/chosen": -1.0570118427276611, + "logits/rejected": -1.0660746097564697, + "logps/chosen": -655.458251953125, + "logps/rejected": -789.9827880859375, + "loss": 0.311, + "rewards/accuracies": 0.9375, + "rewards/chosen": -4.035057544708252, + "rewards/margins": 1.403611183166504, + "rewards/rejected": -5.438669204711914, + "step": 463 + }, + { + "epoch": 0.9917178733636121, + "grad_norm": 30.473984400030304, + "learning_rate": 8.811777012271427e-11, + "logits/chosen": -1.2154146432876587, + "logits/rejected": -1.2058013677597046, + "logps/chosen": -639.0227661132812, + "logps/rejected": -820.2091064453125, + "loss": 0.3856, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.121957778930664, + "rewards/margins": 1.6997212171554565, + "rewards/rejected": -5.821678638458252, + "step": 464 + }, + { + "epoch": 0.9938551963665508, + "grad_norm": 23.82578403366403, + "learning_rate": 3.9164366404298435e-11, + "logits/chosen": -1.0110024213790894, + "logits/rejected": -1.0080432891845703, + "logps/chosen": -551.5545654296875, + "logps/rejected": -713.6527099609375, + "loss": 0.3884, + "rewards/accuracies": 0.875, + "rewards/chosen": -3.4868009090423584, + "rewards/margins": 1.5603687763214111, + "rewards/rejected": -5.047170162200928, + "step": 465 + }, + { + "epoch": 0.9959925193694897, + "grad_norm": 23.209923292158702, + "learning_rate": 9.791228555566799e-12, + "logits/chosen": -0.9966657161712646, + "logits/rejected": -1.022011160850525, + "logps/chosen": -511.430908203125, + "logps/rejected": -595.9152221679688, + "loss": 0.397, + "rewards/accuracies": 0.6875, + "rewards/chosen": -3.4004158973693848, + "rewards/margins": 0.7674331068992615, + "rewards/rejected": -4.167849063873291, + "step": 466 + }, + { + "epoch": 0.9981298423724285, + "grad_norm": 21.729353783070525, + "learning_rate": 0.0, + "logits/chosen": -1.0969784259796143, + "logits/rejected": -1.0713554620742798, + "logps/chosen": -821.9149780273438, + "logps/rejected": -1118.57958984375, + "loss": 0.3546, + "rewards/accuracies": 0.8125, + "rewards/chosen": -4.988923072814941, + "rewards/margins": 3.1074531078338623, + "rewards/rejected": -8.096376419067383, + "step": 467 + }, + { + "epoch": 0.9981298423724285, + "step": 467, + "total_flos": 0.0, + "train_loss": 0.4754098514673286, + "train_runtime": 9903.4254, + "train_samples_per_second": 6.046, + "train_steps_per_second": 0.047 + } + ], + "logging_steps": 1, + "max_steps": 467, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}