diff --git "a/Gen_model_Minicpm_2.4b/trainer_state.json" "b/Gen_model_Minicpm_2.4b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/Gen_model_Minicpm_2.4b/trainer_state.json" @@ -0,0 +1,4337 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9282399143163156, + "eval_steps": 100, + "global_step": 2600, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.003570153516601214, + "grad_norm": 7.480371952056885, + "learning_rate": 4.9821492324169946e-05, + "logits/chosen": 24.812057495117188, + "logits/rejected": 24.878061294555664, + "logps/chosen": -20.12216567993164, + "logps/rejected": -25.59366798400879, + "loss": 0.6569, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.08925116807222366, + "rewards/margins": 0.21577796339988708, + "rewards/rejected": -0.12652680277824402, + "step": 10 + }, + { + "epoch": 0.007140307033202428, + "grad_norm": 6.517239093780518, + "learning_rate": 4.964298464833988e-05, + "logits/chosen": 24.764490127563477, + "logits/rejected": 24.641494750976562, + "logps/chosen": -11.448348999023438, + "logps/rejected": -12.761981964111328, + "loss": 0.6509, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.14247111976146698, + "rewards/margins": 0.24720080196857452, + "rewards/rejected": -0.10472966730594635, + "step": 20 + }, + { + "epoch": 0.010710460549803642, + "grad_norm": 6.2558159828186035, + "learning_rate": 4.946447697250982e-05, + "logits/chosen": 26.845245361328125, + "logits/rejected": 26.89157485961914, + "logps/chosen": -32.96189880371094, + "logps/rejected": -36.98007583618164, + "loss": 0.7495, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.26471006870269775, + "rewards/margins": 0.1922953575849533, + "rewards/rejected": -0.45700541138648987, + "step": 30 + }, + { + "epoch": 0.014280614066404856, + "grad_norm": 7.402151584625244, + "learning_rate": 4.928596929667976e-05, + "logits/chosen": 27.013961791992188, + "logits/rejected": 27.773513793945312, + "logps/chosen": -33.79491424560547, + "logps/rejected": -45.16600799560547, + "loss": 0.607, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.004278683569282293, + "rewards/margins": 0.5446106195449829, + "rewards/rejected": -0.5488892793655396, + "step": 40 + }, + { + "epoch": 0.01785076758300607, + "grad_norm": 8.540044784545898, + "learning_rate": 4.9107461620849696e-05, + "logits/chosen": 24.013198852539062, + "logits/rejected": 24.5129337310791, + "logps/chosen": -45.71046829223633, + "logps/rejected": -38.64923858642578, + "loss": 0.5787, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.3722326159477234, + "rewards/margins": 0.14807865023612976, + "rewards/rejected": -0.5203112363815308, + "step": 50 + }, + { + "epoch": 0.021420921099607283, + "grad_norm": 4.945995330810547, + "learning_rate": 4.892895394501964e-05, + "logits/chosen": 27.04281997680664, + "logits/rejected": 26.753490447998047, + "logps/chosen": -58.52327346801758, + "logps/rejected": -42.7177619934082, + "loss": 0.6459, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.732448399066925, + "rewards/margins": 0.44616565108299255, + "rewards/rejected": -1.1786140203475952, + "step": 60 + }, + { + "epoch": 0.024991074616208496, + "grad_norm": 7.011103630065918, + "learning_rate": 4.875044626918958e-05, + "logits/chosen": 25.751834869384766, + "logits/rejected": 26.487178802490234, + "logps/chosen": -59.837867736816406, + "logps/rejected": -62.329681396484375, + "loss": 0.6127, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.34982752799987793, + "rewards/margins": 0.5455743074417114, + "rewards/rejected": -0.8954018354415894, + "step": 70 + }, + { + "epoch": 0.028561228132809712, + "grad_norm": 4.859188556671143, + "learning_rate": 4.8571938593359514e-05, + "logits/chosen": 23.81821060180664, + "logits/rejected": 23.5118465423584, + "logps/chosen": -42.71343231201172, + "logps/rejected": -37.387977600097656, + "loss": 0.5896, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.011353443376719952, + "rewards/margins": 0.47227078676223755, + "rewards/rejected": -0.4836241602897644, + "step": 80 + }, + { + "epoch": 0.03213138164941092, + "grad_norm": 11.05513858795166, + "learning_rate": 4.839343091752946e-05, + "logits/chosen": 27.86802101135254, + "logits/rejected": 27.87044334411621, + "logps/chosen": -56.7071533203125, + "logps/rejected": -48.46919631958008, + "loss": 0.6217, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.01455842237919569, + "rewards/margins": 0.41877785325050354, + "rewards/rejected": -0.4333363175392151, + "step": 90 + }, + { + "epoch": 0.03570153516601214, + "grad_norm": 5.356033802032471, + "learning_rate": 4.8214923241699396e-05, + "logits/chosen": 23.89133644104004, + "logits/rejected": 23.665489196777344, + "logps/chosen": -63.13393020629883, + "logps/rejected": -43.369056701660156, + "loss": 0.6764, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.114576056599617, + "rewards/margins": 0.36550942063331604, + "rewards/rejected": -0.48008546233177185, + "step": 100 + }, + { + "epoch": 0.03570153516601214, + "eval_logits/chosen": 25.279199600219727, + "eval_logits/rejected": 25.355634689331055, + "eval_logps/chosen": -42.37962341308594, + "eval_logps/rejected": -37.090999603271484, + "eval_loss": 0.5741721987724304, + "eval_rewards/accuracies": 0.6777777671813965, + "eval_rewards/chosen": -0.07397282123565674, + "eval_rewards/margins": 0.43140149116516113, + "eval_rewards/rejected": -0.5053743124008179, + "eval_runtime": 109.7281, + "eval_samples_per_second": 13.05, + "eval_steps_per_second": 0.82, + "step": 100 + }, + { + "epoch": 0.039271688682613354, + "grad_norm": 5.330838680267334, + "learning_rate": 4.803641556586933e-05, + "logits/chosen": 26.816951751708984, + "logits/rejected": 27.308795928955078, + "logps/chosen": -37.895626068115234, + "logps/rejected": -35.43812942504883, + "loss": 0.5487, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.1233404278755188, + "rewards/margins": 0.47418642044067383, + "rewards/rejected": -0.5975269079208374, + "step": 110 + }, + { + "epoch": 0.042841842199214566, + "grad_norm": 10.010088920593262, + "learning_rate": 4.785790789003927e-05, + "logits/chosen": 27.359798431396484, + "logits/rejected": 27.50588035583496, + "logps/chosen": -52.02180862426758, + "logps/rejected": -48.30536651611328, + "loss": 0.5954, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.14739781618118286, + "rewards/margins": 0.22387762367725372, + "rewards/rejected": -0.07647980749607086, + "step": 120 + }, + { + "epoch": 0.04641199571581578, + "grad_norm": 7.994888782501221, + "learning_rate": 4.7679400214209214e-05, + "logits/chosen": 27.39333152770996, + "logits/rejected": 26.996658325195312, + "logps/chosen": -18.88664436340332, + "logps/rejected": -39.95366668701172, + "loss": 0.6306, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.09302469342947006, + "rewards/margins": 0.047691889107227325, + "rewards/rejected": -0.14071659743785858, + "step": 130 + }, + { + "epoch": 0.04998214923241699, + "grad_norm": 3.463163375854492, + "learning_rate": 4.750089253837915e-05, + "logits/chosen": 22.470857620239258, + "logits/rejected": 22.843807220458984, + "logps/chosen": -22.85379981994629, + "logps/rejected": -19.81589698791504, + "loss": 0.5663, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.08994263410568237, + "rewards/margins": 0.6263381242752075, + "rewards/rejected": -0.5363954305648804, + "step": 140 + }, + { + "epoch": 0.053552302749018205, + "grad_norm": 7.127109527587891, + "learning_rate": 4.732238486254909e-05, + "logits/chosen": 26.753835678100586, + "logits/rejected": 26.78439712524414, + "logps/chosen": -46.176551818847656, + "logps/rejected": -50.23821258544922, + "loss": 0.5586, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20682087540626526, + "rewards/margins": 1.0148881673812866, + "rewards/rejected": -0.8080674409866333, + "step": 150 + }, + { + "epoch": 0.057122456265619424, + "grad_norm": 6.003675937652588, + "learning_rate": 4.714387718671903e-05, + "logits/chosen": 29.764068603515625, + "logits/rejected": 29.705978393554688, + "logps/chosen": -23.824987411499023, + "logps/rejected": -42.840850830078125, + "loss": 0.5256, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.22749033570289612, + "rewards/margins": 0.35782289505004883, + "rewards/rejected": -0.5853132009506226, + "step": 160 + }, + { + "epoch": 0.06069260978222064, + "grad_norm": 6.028194427490234, + "learning_rate": 4.696536951088897e-05, + "logits/chosen": 23.769296646118164, + "logits/rejected": 24.033222198486328, + "logps/chosen": -41.425865173339844, + "logps/rejected": -41.90104293823242, + "loss": 0.6533, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.4930092692375183, + "rewards/margins": 0.38698917627334595, + "rewards/rejected": -0.8799983859062195, + "step": 170 + }, + { + "epoch": 0.06426276329882184, + "grad_norm": 4.039268493652344, + "learning_rate": 4.678686183505891e-05, + "logits/chosen": 25.150955200195312, + "logits/rejected": 25.3060302734375, + "logps/chosen": -33.606788635253906, + "logps/rejected": -22.799793243408203, + "loss": 0.5998, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12118367105722427, + "rewards/margins": 0.29234248399734497, + "rewards/rejected": -0.41352614760398865, + "step": 180 + }, + { + "epoch": 0.06783291681542307, + "grad_norm": 8.811986923217773, + "learning_rate": 4.660835415922885e-05, + "logits/chosen": 28.551767349243164, + "logits/rejected": 28.756189346313477, + "logps/chosen": -45.27893829345703, + "logps/rejected": -28.83548355102539, + "loss": 0.5604, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.4300183653831482, + "rewards/margins": 0.17230427265167236, + "rewards/rejected": -0.6023226976394653, + "step": 190 + }, + { + "epoch": 0.07140307033202428, + "grad_norm": 7.31522798538208, + "learning_rate": 4.642984648339879e-05, + "logits/chosen": 25.931976318359375, + "logits/rejected": 25.951126098632812, + "logps/chosen": -50.394126892089844, + "logps/rejected": -38.347755432128906, + "loss": 0.6238, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.2471456527709961, + "rewards/margins": 0.6917206048965454, + "rewards/rejected": -0.9388662576675415, + "step": 200 + }, + { + "epoch": 0.07140307033202428, + "eval_logits/chosen": 25.649152755737305, + "eval_logits/rejected": 25.710378646850586, + "eval_logps/chosen": -43.097450256347656, + "eval_logps/rejected": -38.84811782836914, + "eval_loss": 0.5490943193435669, + "eval_rewards/accuracies": 0.6833333373069763, + "eval_rewards/chosen": -0.14575596153736115, + "eval_rewards/margins": 0.5353304743766785, + "eval_rewards/rejected": -0.6810863614082336, + "eval_runtime": 97.7698, + "eval_samples_per_second": 14.647, + "eval_steps_per_second": 0.921, + "step": 200 + }, + { + "epoch": 0.0749732238486255, + "grad_norm": 5.679516792297363, + "learning_rate": 4.6251338807568726e-05, + "logits/chosen": 26.078624725341797, + "logits/rejected": 25.89596939086914, + "logps/chosen": -28.943302154541016, + "logps/rejected": -45.90317916870117, + "loss": 0.6111, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14115412533283234, + "rewards/margins": 0.2540377974510193, + "rewards/rejected": -0.3951919376850128, + "step": 210 + }, + { + "epoch": 0.07854337736522671, + "grad_norm": 13.459559440612793, + "learning_rate": 4.607283113173867e-05, + "logits/chosen": 22.200332641601562, + "logits/rejected": 22.71112060546875, + "logps/chosen": -36.98976135253906, + "logps/rejected": -30.201828002929688, + "loss": 0.5806, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.00881283264607191, + "rewards/margins": 0.8385305404663086, + "rewards/rejected": -0.8473433256149292, + "step": 220 + }, + { + "epoch": 0.08211353088182792, + "grad_norm": 9.37973403930664, + "learning_rate": 4.589432345590861e-05, + "logits/chosen": 28.823495864868164, + "logits/rejected": 28.9989013671875, + "logps/chosen": -52.96837615966797, + "logps/rejected": -48.45201110839844, + "loss": 0.5957, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.45236507058143616, + "rewards/margins": 0.9013611674308777, + "rewards/rejected": -1.3537262678146362, + "step": 230 + }, + { + "epoch": 0.08568368439842913, + "grad_norm": 6.611434459686279, + "learning_rate": 4.5715815780078545e-05, + "logits/chosen": 19.64382553100586, + "logits/rejected": 19.509660720825195, + "logps/chosen": -48.31082534790039, + "logps/rejected": -50.42847442626953, + "loss": 0.6004, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.15059544146060944, + "rewards/margins": 0.887191653251648, + "rewards/rejected": -1.0377871990203857, + "step": 240 + }, + { + "epoch": 0.08925383791503035, + "grad_norm": 6.911257266998291, + "learning_rate": 4.553730810424848e-05, + "logits/chosen": 17.858783721923828, + "logits/rejected": 18.209197998046875, + "logps/chosen": -28.553043365478516, + "logps/rejected": -39.740516662597656, + "loss": 0.5548, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.13874831795692444, + "rewards/margins": 0.48837804794311523, + "rewards/rejected": -0.6271263360977173, + "step": 250 + }, + { + "epoch": 0.09282399143163156, + "grad_norm": 8.965351104736328, + "learning_rate": 4.5358800428418426e-05, + "logits/chosen": 25.02800178527832, + "logits/rejected": 24.903820037841797, + "logps/chosen": -41.33483123779297, + "logps/rejected": -38.99028396606445, + "loss": 0.5356, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.14857253432273865, + "rewards/margins": 0.6738235354423523, + "rewards/rejected": -0.525251030921936, + "step": 260 + }, + { + "epoch": 0.09639414494823277, + "grad_norm": 6.815878868103027, + "learning_rate": 4.5180292752588364e-05, + "logits/chosen": 26.964553833007812, + "logits/rejected": 26.749637603759766, + "logps/chosen": -36.553131103515625, + "logps/rejected": -19.335412979125977, + "loss": 0.5599, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.019418681040406227, + "rewards/margins": 0.6367809772491455, + "rewards/rejected": -0.6561996936798096, + "step": 270 + }, + { + "epoch": 0.09996429846483398, + "grad_norm": 9.245991706848145, + "learning_rate": 4.50017850767583e-05, + "logits/chosen": 25.099380493164062, + "logits/rejected": 25.418010711669922, + "logps/chosen": -30.81514549255371, + "logps/rejected": -27.115314483642578, + "loss": 0.6413, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.4499152600765228, + "rewards/margins": 0.7015028595924377, + "rewards/rejected": -1.1514180898666382, + "step": 280 + }, + { + "epoch": 0.1035344519814352, + "grad_norm": 4.70521879196167, + "learning_rate": 4.4823277400928245e-05, + "logits/chosen": 26.4927978515625, + "logits/rejected": 26.206491470336914, + "logps/chosen": -50.66899871826172, + "logps/rejected": -31.35797691345215, + "loss": 0.5381, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.7465568780899048, + "rewards/margins": 0.4888392388820648, + "rewards/rejected": -1.235396146774292, + "step": 290 + }, + { + "epoch": 0.10710460549803641, + "grad_norm": 12.274664878845215, + "learning_rate": 4.464476972509818e-05, + "logits/chosen": 24.71265983581543, + "logits/rejected": 24.88272476196289, + "logps/chosen": -37.392120361328125, + "logps/rejected": -31.838241577148438, + "loss": 0.6747, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.5502563714981079, + "rewards/margins": 0.21214011311531067, + "rewards/rejected": -0.762396514415741, + "step": 300 + }, + { + "epoch": 0.10710460549803641, + "eval_logits/chosen": 24.666624069213867, + "eval_logits/rejected": 24.732454299926758, + "eval_logps/chosen": -43.291751861572266, + "eval_logps/rejected": -40.54462432861328, + "eval_loss": 0.5352813601493835, + "eval_rewards/accuracies": 0.7222222089767456, + "eval_rewards/chosen": -0.16518594324588776, + "eval_rewards/margins": 0.6855509877204895, + "eval_rewards/rejected": -0.8507369160652161, + "eval_runtime": 97.8407, + "eval_samples_per_second": 14.636, + "eval_steps_per_second": 0.92, + "step": 300 + }, + { + "epoch": 0.11067475901463764, + "grad_norm": 7.402091979980469, + "learning_rate": 4.446626204926812e-05, + "logits/chosen": 25.197057723999023, + "logits/rejected": 25.214290618896484, + "logps/chosen": -53.834320068359375, + "logps/rejected": -47.46216583251953, + "loss": 0.5246, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.2350492775440216, + "rewards/margins": 0.7798230051994324, + "rewards/rejected": -1.0148723125457764, + "step": 310 + }, + { + "epoch": 0.11424491253123885, + "grad_norm": 7.803309917449951, + "learning_rate": 4.4287754373438064e-05, + "logits/chosen": 25.673919677734375, + "logits/rejected": 26.028600692749023, + "logps/chosen": -47.897422790527344, + "logps/rejected": -32.04759216308594, + "loss": 0.5493, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.12382154166698456, + "rewards/margins": 1.0582133531570435, + "rewards/rejected": -0.934391975402832, + "step": 320 + }, + { + "epoch": 0.11781506604784006, + "grad_norm": 12.149727821350098, + "learning_rate": 4.4109246697608e-05, + "logits/chosen": 26.565237045288086, + "logits/rejected": 26.2646541595459, + "logps/chosen": -45.204307556152344, + "logps/rejected": -54.38944625854492, + "loss": 0.5822, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.24066975712776184, + "rewards/margins": 0.3767842650413513, + "rewards/rejected": -0.6174539923667908, + "step": 330 + }, + { + "epoch": 0.12138521956444127, + "grad_norm": 11.431114196777344, + "learning_rate": 4.393073902177794e-05, + "logits/chosen": 24.764408111572266, + "logits/rejected": 23.79238510131836, + "logps/chosen": -22.893648147583008, + "logps/rejected": -48.49061584472656, + "loss": 0.6424, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.6392322778701782, + "rewards/margins": 0.1874510645866394, + "rewards/rejected": -0.8266833424568176, + "step": 340 + }, + { + "epoch": 0.12495537308104249, + "grad_norm": 13.246600151062012, + "learning_rate": 4.375223134594788e-05, + "logits/chosen": 23.024621963500977, + "logits/rejected": 22.838958740234375, + "logps/chosen": -46.08310317993164, + "logps/rejected": -63.932945251464844, + "loss": 0.5871, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.7620424032211304, + "rewards/margins": 0.4331347346305847, + "rewards/rejected": -1.1951771974563599, + "step": 350 + }, + { + "epoch": 0.12852552659764369, + "grad_norm": 6.875857830047607, + "learning_rate": 4.357372367011782e-05, + "logits/chosen": 24.33723258972168, + "logits/rejected": 24.633174896240234, + "logps/chosen": -36.276363372802734, + "logps/rejected": -41.762107849121094, + "loss": 0.538, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5144018530845642, + "rewards/margins": 0.7962431907653809, + "rewards/rejected": -1.3106449842453003, + "step": 360 + }, + { + "epoch": 0.1320956801142449, + "grad_norm": 8.577788352966309, + "learning_rate": 4.339521599428776e-05, + "logits/chosen": 18.497318267822266, + "logits/rejected": 18.542171478271484, + "logps/chosen": -54.802284240722656, + "logps/rejected": -40.361900329589844, + "loss": 0.5472, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.43400055170059204, + "rewards/margins": 0.37733301520347595, + "rewards/rejected": -0.8113336563110352, + "step": 370 + }, + { + "epoch": 0.13566583363084614, + "grad_norm": 7.8961052894592285, + "learning_rate": 4.3216708318457694e-05, + "logits/chosen": 20.79564094543457, + "logits/rejected": 21.202953338623047, + "logps/chosen": -46.6911735534668, + "logps/rejected": -25.177892684936523, + "loss": 0.5404, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.20933791995048523, + "rewards/margins": 0.9722681045532227, + "rewards/rejected": -1.1816058158874512, + "step": 380 + }, + { + "epoch": 0.13923598714744734, + "grad_norm": 6.78292989730835, + "learning_rate": 4.303820064262764e-05, + "logits/chosen": 24.46206283569336, + "logits/rejected": 24.468271255493164, + "logps/chosen": -17.142438888549805, + "logps/rejected": -8.94920539855957, + "loss": 0.5916, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5404640436172485, + "rewards/margins": -0.012588715180754662, + "rewards/rejected": -0.5278753042221069, + "step": 390 + }, + { + "epoch": 0.14280614066404856, + "grad_norm": 6.525446891784668, + "learning_rate": 4.2859692966797575e-05, + "logits/chosen": 27.94113540649414, + "logits/rejected": 27.818674087524414, + "logps/chosen": -28.574228286743164, + "logps/rejected": -41.67361831665039, + "loss": 0.5795, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17134228348731995, + "rewards/margins": 0.6803145408630371, + "rewards/rejected": -0.8516567945480347, + "step": 400 + }, + { + "epoch": 0.14280614066404856, + "eval_logits/chosen": 23.894601821899414, + "eval_logits/rejected": 23.941455841064453, + "eval_logps/chosen": -42.414432525634766, + "eval_logps/rejected": -40.24538040161133, + "eval_loss": 0.5327069759368896, + "eval_rewards/accuracies": 0.7944444417953491, + "eval_rewards/chosen": -0.07745402306318283, + "eval_rewards/margins": 0.7433586716651917, + "eval_rewards/rejected": -0.8208127617835999, + "eval_runtime": 97.7382, + "eval_samples_per_second": 14.651, + "eval_steps_per_second": 0.921, + "step": 400 + }, + { + "epoch": 0.14637629418064976, + "grad_norm": 7.846954345703125, + "learning_rate": 4.268118529096751e-05, + "logits/chosen": 25.257925033569336, + "logits/rejected": 25.105932235717773, + "logps/chosen": -45.8572883605957, + "logps/rejected": -26.36654281616211, + "loss": 0.4999, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.5164770483970642, + "rewards/margins": 0.7112953662872314, + "rewards/rejected": -1.2277723550796509, + "step": 410 + }, + { + "epoch": 0.149946447697251, + "grad_norm": 7.722918510437012, + "learning_rate": 4.250267761513746e-05, + "logits/chosen": 21.332164764404297, + "logits/rejected": 21.323444366455078, + "logps/chosen": -37.462974548339844, + "logps/rejected": -28.17437171936035, + "loss": 0.6035, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.4826648235321045, + "rewards/margins": 0.2844996452331543, + "rewards/rejected": -0.7671645283699036, + "step": 420 + }, + { + "epoch": 0.1535166012138522, + "grad_norm": 8.375965118408203, + "learning_rate": 4.232416993930739e-05, + "logits/chosen": 18.349498748779297, + "logits/rejected": 18.25157356262207, + "logps/chosen": -53.23881912231445, + "logps/rejected": -62.28485107421875, + "loss": 0.5301, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14654412865638733, + "rewards/margins": 0.404157817363739, + "rewards/rejected": -0.550701916217804, + "step": 430 + }, + { + "epoch": 0.15708675473045342, + "grad_norm": 6.275489807128906, + "learning_rate": 4.214566226347733e-05, + "logits/chosen": 22.851360321044922, + "logits/rejected": 23.03590202331543, + "logps/chosen": -32.48253631591797, + "logps/rejected": -36.65414047241211, + "loss": 0.4959, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13281583786010742, + "rewards/margins": 1.3949477672576904, + "rewards/rejected": -1.262131929397583, + "step": 440 + }, + { + "epoch": 0.16065690824705461, + "grad_norm": 9.541354179382324, + "learning_rate": 4.1967154587647275e-05, + "logits/chosen": 25.5571231842041, + "logits/rejected": 25.35956573486328, + "logps/chosen": -33.83552169799805, + "logps/rejected": -34.80134963989258, + "loss": 0.5615, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1620870977640152, + "rewards/margins": 1.014452576637268, + "rewards/rejected": -1.176539659500122, + "step": 450 + }, + { + "epoch": 0.16422706176365584, + "grad_norm": 4.77647590637207, + "learning_rate": 4.1788646911817206e-05, + "logits/chosen": 25.47999382019043, + "logits/rejected": 24.808551788330078, + "logps/chosen": -39.025047302246094, + "logps/rejected": -71.5890884399414, + "loss": 0.4752, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.2537313401699066, + "rewards/margins": 1.16329026222229, + "rewards/rejected": -1.4170215129852295, + "step": 460 + }, + { + "epoch": 0.16779721528025704, + "grad_norm": 4.253603458404541, + "learning_rate": 4.161013923598715e-05, + "logits/chosen": 20.190471649169922, + "logits/rejected": 20.048721313476562, + "logps/chosen": -68.62611389160156, + "logps/rejected": -72.08489227294922, + "loss": 0.5801, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9637469053268433, + "rewards/margins": 0.6993335485458374, + "rewards/rejected": -1.6630805730819702, + "step": 470 + }, + { + "epoch": 0.17136736879685827, + "grad_norm": 9.119401931762695, + "learning_rate": 4.1431631560157094e-05, + "logits/chosen": 19.260848999023438, + "logits/rejected": 18.897022247314453, + "logps/chosen": -39.502891540527344, + "logps/rejected": -37.60371398925781, + "loss": 0.6457, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7391226291656494, + "rewards/margins": 0.16527147591114044, + "rewards/rejected": -0.9043940305709839, + "step": 480 + }, + { + "epoch": 0.1749375223134595, + "grad_norm": 6.5403876304626465, + "learning_rate": 4.1253123884327025e-05, + "logits/chosen": 24.00229263305664, + "logits/rejected": 24.137210845947266, + "logps/chosen": -45.11564636230469, + "logps/rejected": -63.3198356628418, + "loss": 0.492, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.5857070684432983, + "rewards/margins": 1.3458783626556396, + "rewards/rejected": -1.9315853118896484, + "step": 490 + }, + { + "epoch": 0.1785076758300607, + "grad_norm": 6.147622108459473, + "learning_rate": 4.107461620849697e-05, + "logits/chosen": 20.132909774780273, + "logits/rejected": 20.044574737548828, + "logps/chosen": -40.866424560546875, + "logps/rejected": -50.83662414550781, + "loss": 0.4606, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.40833067893981934, + "rewards/margins": 0.7871369123458862, + "rewards/rejected": -1.195467472076416, + "step": 500 + }, + { + "epoch": 0.1785076758300607, + "eval_logits/chosen": 22.78708839416504, + "eval_logits/rejected": 22.809024810791016, + "eval_logps/chosen": -47.07508850097656, + "eval_logps/rejected": -45.6721076965332, + "eval_loss": 0.5191536545753479, + "eval_rewards/accuracies": 0.75, + "eval_rewards/chosen": -0.5435196161270142, + "eval_rewards/margins": 0.8199654221534729, + "eval_rewards/rejected": -1.3634849786758423, + "eval_runtime": 97.7087, + "eval_samples_per_second": 14.656, + "eval_steps_per_second": 0.921, + "step": 500 + }, + { + "epoch": 0.18207782934666192, + "grad_norm": 12.392146110534668, + "learning_rate": 4.0896108532666906e-05, + "logits/chosen": 19.699203491210938, + "logits/rejected": 19.52513885498047, + "logps/chosen": -57.56768798828125, + "logps/rejected": -71.2248764038086, + "loss": 0.632, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.3169898986816406, + "rewards/margins": 1.7359142303466797, + "rewards/rejected": -2.0529043674468994, + "step": 510 + }, + { + "epoch": 0.18564798286326312, + "grad_norm": 5.329209804534912, + "learning_rate": 4.071760085683684e-05, + "logits/chosen": 20.416704177856445, + "logits/rejected": 20.525035858154297, + "logps/chosen": -38.03459930419922, + "logps/rejected": -38.064720153808594, + "loss": 0.5251, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.5582042932510376, + "rewards/margins": 0.7457180023193359, + "rewards/rejected": -1.3039222955703735, + "step": 520 + }, + { + "epoch": 0.18921813637986434, + "grad_norm": 5.097146987915039, + "learning_rate": 4.053909318100679e-05, + "logits/chosen": 22.233572006225586, + "logits/rejected": 22.370027542114258, + "logps/chosen": -15.693742752075195, + "logps/rejected": -16.01146125793457, + "loss": 0.4981, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.28356021642684937, + "rewards/margins": 0.759593665599823, + "rewards/rejected": -1.0431538820266724, + "step": 530 + }, + { + "epoch": 0.19278828989646554, + "grad_norm": 10.173523902893066, + "learning_rate": 4.0360585505176725e-05, + "logits/chosen": 22.36469268798828, + "logits/rejected": 22.462005615234375, + "logps/chosen": -22.046735763549805, + "logps/rejected": -29.99558448791504, + "loss": 0.5417, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.5353186726570129, + "rewards/margins": 0.14299364387989044, + "rewards/rejected": -0.6783123016357422, + "step": 540 + }, + { + "epoch": 0.19635844341306677, + "grad_norm": 4.725411891937256, + "learning_rate": 4.018207782934666e-05, + "logits/chosen": 22.934490203857422, + "logits/rejected": 22.83697509765625, + "logps/chosen": -50.47562789916992, + "logps/rejected": -39.98368453979492, + "loss": 0.656, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.5939350724220276, + "rewards/margins": 0.7087148427963257, + "rewards/rejected": -1.3026498556137085, + "step": 550 + }, + { + "epoch": 0.19992859692966797, + "grad_norm": 16.693065643310547, + "learning_rate": 4.00035701535166e-05, + "logits/chosen": 26.147781372070312, + "logits/rejected": 25.938640594482422, + "logps/chosen": -66.49282836914062, + "logps/rejected": -62.023353576660156, + "loss": 0.5596, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1642974466085434, + "rewards/margins": 1.2225383520126343, + "rewards/rejected": -1.3868358135223389, + "step": 560 + }, + { + "epoch": 0.2034987504462692, + "grad_norm": 4.374217987060547, + "learning_rate": 3.982506247768654e-05, + "logits/chosen": 23.22292709350586, + "logits/rejected": 22.756343841552734, + "logps/chosen": -47.733238220214844, + "logps/rejected": -65.7103042602539, + "loss": 0.5128, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.5647065043449402, + "rewards/margins": 0.45924702286720276, + "rewards/rejected": -1.0239535570144653, + "step": 570 + }, + { + "epoch": 0.2070689039628704, + "grad_norm": 5.655895233154297, + "learning_rate": 3.964655480185648e-05, + "logits/chosen": 22.537179946899414, + "logits/rejected": 22.3128662109375, + "logps/chosen": -33.57523727416992, + "logps/rejected": -46.982025146484375, + "loss": 0.485, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.5356764793395996, + "rewards/margins": 1.0464246273040771, + "rewards/rejected": -1.5821009874343872, + "step": 580 + }, + { + "epoch": 0.21063905747947162, + "grad_norm": 8.67690372467041, + "learning_rate": 3.946804712602642e-05, + "logits/chosen": 24.053617477416992, + "logits/rejected": 24.086387634277344, + "logps/chosen": -30.85910415649414, + "logps/rejected": -17.82439422607422, + "loss": 0.4658, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6462145447731018, + "rewards/margins": 0.6275963187217712, + "rewards/rejected": -1.273810863494873, + "step": 590 + }, + { + "epoch": 0.21420921099607282, + "grad_norm": 12.361294746398926, + "learning_rate": 3.928953945019636e-05, + "logits/chosen": 19.749366760253906, + "logits/rejected": 20.843948364257812, + "logps/chosen": -47.06502151489258, + "logps/rejected": -38.376441955566406, + "loss": 0.4708, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.37672120332717896, + "rewards/margins": 1.4848692417144775, + "rewards/rejected": -1.8615906238555908, + "step": 600 + }, + { + "epoch": 0.21420921099607282, + "eval_logits/chosen": 22.7338924407959, + "eval_logits/rejected": 22.768844604492188, + "eval_logps/chosen": -43.45345687866211, + "eval_logps/rejected": -43.90707778930664, + "eval_loss": 0.5074149966239929, + "eval_rewards/accuracies": 0.7944444417953491, + "eval_rewards/chosen": -0.18135611712932587, + "eval_rewards/margins": 1.0056260824203491, + "eval_rewards/rejected": -1.1869820356369019, + "eval_runtime": 97.81, + "eval_samples_per_second": 14.641, + "eval_steps_per_second": 0.92, + "step": 600 + }, + { + "epoch": 0.21777936451267405, + "grad_norm": 11.45566177368164, + "learning_rate": 3.91110317743663e-05, + "logits/chosen": 17.696277618408203, + "logits/rejected": 18.161190032958984, + "logps/chosen": -61.38924026489258, + "logps/rejected": -28.698049545288086, + "loss": 0.6026, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.9514519572257996, + "rewards/margins": -0.2889273166656494, + "rewards/rejected": -0.6625245809555054, + "step": 610 + }, + { + "epoch": 0.22134951802927527, + "grad_norm": 11.354263305664062, + "learning_rate": 3.8932524098536237e-05, + "logits/chosen": 23.74215316772461, + "logits/rejected": 23.932151794433594, + "logps/chosen": -39.733665466308594, + "logps/rejected": -52.69596481323242, + "loss": 0.5451, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11807258427143097, + "rewards/margins": 0.5899603962898254, + "rewards/rejected": -0.7080329656600952, + "step": 620 + }, + { + "epoch": 0.22491967154587647, + "grad_norm": 8.663636207580566, + "learning_rate": 3.875401642270618e-05, + "logits/chosen": 20.996337890625, + "logits/rejected": 21.191204071044922, + "logps/chosen": -42.43983840942383, + "logps/rejected": -30.458621978759766, + "loss": 0.5347, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.20720069110393524, + "rewards/margins": 0.8300245404243469, + "rewards/rejected": -1.0372252464294434, + "step": 630 + }, + { + "epoch": 0.2284898250624777, + "grad_norm": 6.389614105224609, + "learning_rate": 3.857550874687612e-05, + "logits/chosen": 23.66379737854004, + "logits/rejected": 24.068124771118164, + "logps/chosen": -56.00236129760742, + "logps/rejected": -52.503746032714844, + "loss": 0.5791, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09035615622997284, + "rewards/margins": 1.4734671115875244, + "rewards/rejected": -1.5638234615325928, + "step": 640 + }, + { + "epoch": 0.2320599785790789, + "grad_norm": 5.053744792938232, + "learning_rate": 3.8397001071046055e-05, + "logits/chosen": 24.763042449951172, + "logits/rejected": 24.731813430786133, + "logps/chosen": -34.47309494018555, + "logps/rejected": -24.708209991455078, + "loss": 0.5188, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.23519682884216309, + "rewards/margins": 0.7381345629692078, + "rewards/rejected": -0.9733314514160156, + "step": 650 + }, + { + "epoch": 0.23563013209568012, + "grad_norm": 14.828216552734375, + "learning_rate": 3.8218493395216e-05, + "logits/chosen": 23.33843421936035, + "logits/rejected": 23.605775833129883, + "logps/chosen": -38.09233856201172, + "logps/rejected": -36.02016830444336, + "loss": 0.6089, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.3239721357822418, + "rewards/margins": 0.35063061118125916, + "rewards/rejected": -0.6746028661727905, + "step": 660 + }, + { + "epoch": 0.23920028561228132, + "grad_norm": 6.591442108154297, + "learning_rate": 3.8039985719385937e-05, + "logits/chosen": 22.064315795898438, + "logits/rejected": 22.158384323120117, + "logps/chosen": -28.91001319885254, + "logps/rejected": -23.76300048828125, + "loss": 0.5987, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.16806618869304657, + "rewards/margins": 0.20326104760169983, + "rewards/rejected": -0.3713272213935852, + "step": 670 + }, + { + "epoch": 0.24277043912888255, + "grad_norm": 9.390283584594727, + "learning_rate": 3.7861478043555874e-05, + "logits/chosen": 23.296916961669922, + "logits/rejected": 23.495471954345703, + "logps/chosen": -51.231292724609375, + "logps/rejected": -32.36138916015625, + "loss": 0.4838, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.4357140064239502, + "rewards/margins": 0.5686477422714233, + "rewards/rejected": -1.0043617486953735, + "step": 680 + }, + { + "epoch": 0.24634059264548375, + "grad_norm": 7.05921745300293, + "learning_rate": 3.768297036772581e-05, + "logits/chosen": 24.67749786376953, + "logits/rejected": 25.084362030029297, + "logps/chosen": -61.90864944458008, + "logps/rejected": -47.69279861450195, + "loss": 0.6434, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.0670510530471802, + "rewards/margins": 0.16362956166267395, + "rewards/rejected": -1.2306805849075317, + "step": 690 + }, + { + "epoch": 0.24991074616208497, + "grad_norm": 6.927039623260498, + "learning_rate": 3.7504462691895755e-05, + "logits/chosen": 22.120304107666016, + "logits/rejected": 22.317134857177734, + "logps/chosen": -47.007747650146484, + "logps/rejected": -29.523571014404297, + "loss": 0.5162, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.47515907883644104, + "rewards/margins": 0.9105289578437805, + "rewards/rejected": -1.385688066482544, + "step": 700 + }, + { + "epoch": 0.24991074616208497, + "eval_logits/chosen": 23.21824073791504, + "eval_logits/rejected": 23.2376651763916, + "eval_logps/chosen": -43.74300765991211, + "eval_logps/rejected": -43.685081481933594, + "eval_loss": 0.5032530426979065, + "eval_rewards/accuracies": 0.7888888716697693, + "eval_rewards/chosen": -0.21031144261360168, + "eval_rewards/margins": 0.9544711709022522, + "eval_rewards/rejected": -1.1647826433181763, + "eval_runtime": 97.9268, + "eval_samples_per_second": 14.623, + "eval_steps_per_second": 0.919, + "step": 700 + }, + { + "epoch": 0.2534808996786862, + "grad_norm": 6.4891767501831055, + "learning_rate": 3.732595501606569e-05, + "logits/chosen": 23.012346267700195, + "logits/rejected": 23.133983612060547, + "logps/chosen": -32.99153137207031, + "logps/rejected": -30.8564453125, + "loss": 0.5311, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.13405704498291016, + "rewards/margins": 0.8106959462165833, + "rewards/rejected": -0.9447528719902039, + "step": 710 + }, + { + "epoch": 0.25705105319528737, + "grad_norm": 7.658117771148682, + "learning_rate": 3.714744734023563e-05, + "logits/chosen": 22.539342880249023, + "logits/rejected": 22.32984161376953, + "logps/chosen": -41.30803298950195, + "logps/rejected": -37.77648162841797, + "loss": 0.583, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.5999592542648315, + "rewards/margins": 0.26343995332717896, + "rewards/rejected": -0.8633992075920105, + "step": 720 + }, + { + "epoch": 0.2606212067118886, + "grad_norm": 7.918398857116699, + "learning_rate": 3.6968939664405574e-05, + "logits/chosen": 22.939983367919922, + "logits/rejected": 23.18207359313965, + "logps/chosen": -46.46357345581055, + "logps/rejected": -35.31732940673828, + "loss": 0.5581, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.043735720217227936, + "rewards/margins": 0.8847159147262573, + "rewards/rejected": -0.928451657295227, + "step": 730 + }, + { + "epoch": 0.2641913602284898, + "grad_norm": 6.20953893661499, + "learning_rate": 3.6790431988575504e-05, + "logits/chosen": 28.148590087890625, + "logits/rejected": 27.938528060913086, + "logps/chosen": -51.41779327392578, + "logps/rejected": -52.49924850463867, + "loss": 0.523, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.0679657459259033, + "rewards/margins": 0.6626083254814148, + "rewards/rejected": -1.7305742502212524, + "step": 740 + }, + { + "epoch": 0.26776151374509105, + "grad_norm": 10.36694049835205, + "learning_rate": 3.661192431274545e-05, + "logits/chosen": 24.841873168945312, + "logits/rejected": 25.12506675720215, + "logps/chosen": -21.518558502197266, + "logps/rejected": -28.731252670288086, + "loss": 0.5559, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3871726393699646, + "rewards/margins": 0.5390633344650269, + "rewards/rejected": -0.9262359738349915, + "step": 750 + }, + { + "epoch": 0.2713316672616923, + "grad_norm": 6.616639137268066, + "learning_rate": 3.643341663691539e-05, + "logits/chosen": 23.483844757080078, + "logits/rejected": 23.462881088256836, + "logps/chosen": -20.33786964416504, + "logps/rejected": -41.595367431640625, + "loss": 0.5861, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.26747018098831177, + "rewards/margins": 0.5755705833435059, + "rewards/rejected": -0.8430407643318176, + "step": 760 + }, + { + "epoch": 0.27490182077829345, + "grad_norm": 7.002477169036865, + "learning_rate": 3.625490896108533e-05, + "logits/chosen": 23.841524124145508, + "logits/rejected": 23.905414581298828, + "logps/chosen": -54.80754852294922, + "logps/rejected": -60.40642166137695, + "loss": 0.5478, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.7754853367805481, + "rewards/margins": 0.7477506995201111, + "rewards/rejected": -1.5232360363006592, + "step": 770 + }, + { + "epoch": 0.2784719742948947, + "grad_norm": 4.716040134429932, + "learning_rate": 3.607640128525527e-05, + "logits/chosen": 21.66192054748535, + "logits/rejected": 21.678348541259766, + "logps/chosen": -59.828773498535156, + "logps/rejected": -51.79111862182617, + "loss": 0.3989, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.5894995927810669, + "rewards/margins": 1.3722957372665405, + "rewards/rejected": -1.9617952108383179, + "step": 780 + }, + { + "epoch": 0.2820421278114959, + "grad_norm": 8.946788787841797, + "learning_rate": 3.589789360942521e-05, + "logits/chosen": 24.498876571655273, + "logits/rejected": 24.739574432373047, + "logps/chosen": -56.08790969848633, + "logps/rejected": -50.47938919067383, + "loss": 0.5822, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.2977168560028076, + "rewards/margins": 0.1555618941783905, + "rewards/rejected": -1.45327889919281, + "step": 790 + }, + { + "epoch": 0.28561228132809713, + "grad_norm": 5.08765983581543, + "learning_rate": 3.571938593359515e-05, + "logits/chosen": 26.44881820678711, + "logits/rejected": 26.592966079711914, + "logps/chosen": -26.545644760131836, + "logps/rejected": -46.947174072265625, + "loss": 0.5121, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5634913444519043, + "rewards/margins": 0.8930961489677429, + "rewards/rejected": -1.456587553024292, + "step": 800 + }, + { + "epoch": 0.28561228132809713, + "eval_logits/chosen": 23.41709327697754, + "eval_logits/rejected": 23.457468032836914, + "eval_logps/chosen": -47.103492736816406, + "eval_logps/rejected": -47.543006896972656, + "eval_loss": 0.4901362657546997, + "eval_rewards/accuracies": 0.7833333611488342, + "eval_rewards/chosen": -0.5463601350784302, + "eval_rewards/margins": 1.0042147636413574, + "eval_rewards/rejected": -1.5505748987197876, + "eval_runtime": 97.7883, + "eval_samples_per_second": 14.644, + "eval_steps_per_second": 0.92, + "step": 800 + }, + { + "epoch": 0.2891824348446983, + "grad_norm": 5.668002605438232, + "learning_rate": 3.5540878257765086e-05, + "logits/chosen": 15.257682800292969, + "logits/rejected": 15.15269947052002, + "logps/chosen": -28.859485626220703, + "logps/rejected": -39.70137023925781, + "loss": 0.6225, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.8887822031974792, + "rewards/margins": 0.40193691849708557, + "rewards/rejected": -1.2907191514968872, + "step": 810 + }, + { + "epoch": 0.2927525883612995, + "grad_norm": 6.119519233703613, + "learning_rate": 3.536237058193502e-05, + "logits/chosen": 20.487186431884766, + "logits/rejected": 20.569347381591797, + "logps/chosen": -26.315814971923828, + "logps/rejected": -24.049549102783203, + "loss": 0.6334, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.2173776626586914, + "rewards/margins": 0.8700194358825684, + "rewards/rejected": -1.0873970985412598, + "step": 820 + }, + { + "epoch": 0.29632274187790075, + "grad_norm": 9.75696849822998, + "learning_rate": 3.518386290610497e-05, + "logits/chosen": 28.570262908935547, + "logits/rejected": 29.040294647216797, + "logps/chosen": -46.7459831237793, + "logps/rejected": -25.238162994384766, + "loss": 0.4854, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.20457518100738525, + "rewards/margins": 1.082558274269104, + "rewards/rejected": -1.2871334552764893, + "step": 830 + }, + { + "epoch": 0.299892895394502, + "grad_norm": 4.077060699462891, + "learning_rate": 3.5005355230274904e-05, + "logits/chosen": 22.926097869873047, + "logits/rejected": 23.03900718688965, + "logps/chosen": -43.289085388183594, + "logps/rejected": -39.03928756713867, + "loss": 0.4416, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.38202500343322754, + "rewards/margins": 0.7754106521606445, + "rewards/rejected": -1.1574357748031616, + "step": 840 + }, + { + "epoch": 0.30346304891110315, + "grad_norm": 6.117227554321289, + "learning_rate": 3.482684755444484e-05, + "logits/chosen": 26.401382446289062, + "logits/rejected": 25.781219482421875, + "logps/chosen": -30.348541259765625, + "logps/rejected": -47.32704162597656, + "loss": 0.4493, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.6118322610855103, + "rewards/margins": 1.2445160150527954, + "rewards/rejected": -1.8563482761383057, + "step": 850 + }, + { + "epoch": 0.3070332024277044, + "grad_norm": 11.844883918762207, + "learning_rate": 3.4648339878614786e-05, + "logits/chosen": 23.075183868408203, + "logits/rejected": 22.080493927001953, + "logps/chosen": -49.947914123535156, + "logps/rejected": -58.764617919921875, + "loss": 0.5217, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.0053069591522217, + "rewards/margins": 0.48744481801986694, + "rewards/rejected": -1.4927518367767334, + "step": 860 + }, + { + "epoch": 0.3106033559443056, + "grad_norm": 12.23678207397461, + "learning_rate": 3.4469832202784716e-05, + "logits/chosen": 26.47686767578125, + "logits/rejected": 26.53334617614746, + "logps/chosen": -27.09230613708496, + "logps/rejected": -39.659324645996094, + "loss": 0.5951, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6957861185073853, + "rewards/margins": 0.6590690016746521, + "rewards/rejected": -1.3548550605773926, + "step": 870 + }, + { + "epoch": 0.31417350946090683, + "grad_norm": 8.197884559631348, + "learning_rate": 3.429132452695466e-05, + "logits/chosen": 22.180370330810547, + "logits/rejected": 22.334489822387695, + "logps/chosen": -45.77663040161133, + "logps/rejected": -41.45575714111328, + "loss": 0.4294, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.40475016832351685, + "rewards/margins": 1.7321338653564453, + "rewards/rejected": -2.1368842124938965, + "step": 880 + }, + { + "epoch": 0.31774366297750806, + "grad_norm": 6.394644260406494, + "learning_rate": 3.4112816851124604e-05, + "logits/chosen": 21.572330474853516, + "logits/rejected": 21.563013076782227, + "logps/chosen": -42.58979034423828, + "logps/rejected": -32.326332092285156, + "loss": 0.5278, + "rewards/accuracies": 0.30000001192092896, + "rewards/chosen": -1.3842201232910156, + "rewards/margins": -0.4129442572593689, + "rewards/rejected": -0.9712759256362915, + "step": 890 + }, + { + "epoch": 0.32131381649410923, + "grad_norm": 11.288488388061523, + "learning_rate": 3.3934309175294535e-05, + "logits/chosen": 23.486909866333008, + "logits/rejected": 23.229293823242188, + "logps/chosen": -26.536273956298828, + "logps/rejected": -35.171104431152344, + "loss": 0.4865, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.39322757720947266, + "rewards/margins": 0.7671092748641968, + "rewards/rejected": -1.160336971282959, + "step": 900 + }, + { + "epoch": 0.32131381649410923, + "eval_logits/chosen": 22.803043365478516, + "eval_logits/rejected": 22.84783172607422, + "eval_logps/chosen": -47.309364318847656, + "eval_logps/rejected": -47.91334533691406, + "eval_loss": 0.4862760305404663, + "eval_rewards/accuracies": 0.7777777910232544, + "eval_rewards/chosen": -0.5669471025466919, + "eval_rewards/margins": 1.0206618309020996, + "eval_rewards/rejected": -1.587609052658081, + "eval_runtime": 97.8776, + "eval_samples_per_second": 14.631, + "eval_steps_per_second": 0.92, + "step": 900 + }, + { + "epoch": 0.32488397001071045, + "grad_norm": 9.65831470489502, + "learning_rate": 3.375580149946448e-05, + "logits/chosen": 26.497304916381836, + "logits/rejected": 26.356307983398438, + "logps/chosen": -28.236690521240234, + "logps/rejected": -34.156455993652344, + "loss": 0.4985, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3999934196472168, + "rewards/margins": 0.484555184841156, + "rewards/rejected": -0.884548544883728, + "step": 910 + }, + { + "epoch": 0.3284541235273117, + "grad_norm": 6.470170021057129, + "learning_rate": 3.357729382363442e-05, + "logits/chosen": 23.454336166381836, + "logits/rejected": 23.263614654541016, + "logps/chosen": -65.40380859375, + "logps/rejected": -39.344261169433594, + "loss": 0.5108, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.46298104524612427, + "rewards/margins": 1.2011882066726685, + "rewards/rejected": -1.6641693115234375, + "step": 920 + }, + { + "epoch": 0.3320242770439129, + "grad_norm": 10.672679901123047, + "learning_rate": 3.3398786147804354e-05, + "logits/chosen": 24.854284286499023, + "logits/rejected": 24.732633590698242, + "logps/chosen": -60.40422821044922, + "logps/rejected": -35.107269287109375, + "loss": 0.5479, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.48133277893066406, + "rewards/margins": 0.2392297238111496, + "rewards/rejected": -0.7205625176429749, + "step": 930 + }, + { + "epoch": 0.3355944305605141, + "grad_norm": 11.122824668884277, + "learning_rate": 3.32202784719743e-05, + "logits/chosen": 25.70233726501465, + "logits/rejected": 25.91379737854004, + "logps/chosen": -45.25204086303711, + "logps/rejected": -32.91322326660156, + "loss": 0.4858, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.7527903318405151, + "rewards/margins": 0.9621620178222656, + "rewards/rejected": -1.7149524688720703, + "step": 940 + }, + { + "epoch": 0.3391645840771153, + "grad_norm": 8.845693588256836, + "learning_rate": 3.3041770796144235e-05, + "logits/chosen": 23.679813385009766, + "logits/rejected": 24.00821304321289, + "logps/chosen": -53.314300537109375, + "logps/rejected": -57.53212356567383, + "loss": 0.4488, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.8183348774909973, + "rewards/margins": 1.1887353658676147, + "rewards/rejected": -2.007070302963257, + "step": 950 + }, + { + "epoch": 0.34273473759371653, + "grad_norm": 11.808037757873535, + "learning_rate": 3.286326312031417e-05, + "logits/chosen": 21.081445693969727, + "logits/rejected": 20.683502197265625, + "logps/chosen": -59.89850997924805, + "logps/rejected": -69.1084213256836, + "loss": 0.5822, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.4903489649295807, + "rewards/margins": 1.016610860824585, + "rewards/rejected": -1.5069600343704224, + "step": 960 + }, + { + "epoch": 0.34630489111031776, + "grad_norm": 4.3298845291137695, + "learning_rate": 3.2684755444484116e-05, + "logits/chosen": 23.46924591064453, + "logits/rejected": 23.299514770507812, + "logps/chosen": -45.10730743408203, + "logps/rejected": -34.44489288330078, + "loss": 0.5883, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9016550779342651, + "rewards/margins": 0.5788153409957886, + "rewards/rejected": -1.4804704189300537, + "step": 970 + }, + { + "epoch": 0.349875044626919, + "grad_norm": 5.846831798553467, + "learning_rate": 3.2506247768654054e-05, + "logits/chosen": 22.685070037841797, + "logits/rejected": 23.413772583007812, + "logps/chosen": -42.309078216552734, + "logps/rejected": -63.73997116088867, + "loss": 0.4721, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.5896610021591187, + "rewards/margins": 1.2323236465454102, + "rewards/rejected": -1.8219846487045288, + "step": 980 + }, + { + "epoch": 0.35344519814352016, + "grad_norm": 8.441866874694824, + "learning_rate": 3.232774009282399e-05, + "logits/chosen": 22.13656997680664, + "logits/rejected": 22.546173095703125, + "logps/chosen": -39.778053283691406, + "logps/rejected": -48.562049865722656, + "loss": 0.5298, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4805465340614319, + "rewards/margins": 1.1920750141143799, + "rewards/rejected": -1.6726213693618774, + "step": 990 + }, + { + "epoch": 0.3570153516601214, + "grad_norm": 8.373982429504395, + "learning_rate": 3.214923241699393e-05, + "logits/chosen": 23.80282974243164, + "logits/rejected": 23.995637893676758, + "logps/chosen": -41.55686950683594, + "logps/rejected": -39.182655334472656, + "loss": 0.6191, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.36075398325920105, + "rewards/margins": 1.393147349357605, + "rewards/rejected": -1.7539011240005493, + "step": 1000 + }, + { + "epoch": 0.3570153516601214, + "eval_logits/chosen": 21.82489013671875, + "eval_logits/rejected": 21.88067054748535, + "eval_logps/chosen": -47.92158126831055, + "eval_logps/rejected": -49.27989196777344, + "eval_loss": 0.474328875541687, + "eval_rewards/accuracies": 0.7888888716697693, + "eval_rewards/chosen": -0.6281694173812866, + "eval_rewards/margins": 1.0960942506790161, + "eval_rewards/rejected": -1.7242639064788818, + "eval_runtime": 97.8094, + "eval_samples_per_second": 14.641, + "eval_steps_per_second": 0.92, + "step": 1000 + }, + { + "epoch": 0.3605855051767226, + "grad_norm": 10.327641487121582, + "learning_rate": 3.197072474116387e-05, + "logits/chosen": 23.04856300354004, + "logits/rejected": 23.33269691467285, + "logps/chosen": -47.23090362548828, + "logps/rejected": -35.351966857910156, + "loss": 0.5248, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.2200241088867188, + "rewards/margins": 0.10034622997045517, + "rewards/rejected": -1.3203704357147217, + "step": 1010 + }, + { + "epoch": 0.36415565869332384, + "grad_norm": 13.408589363098145, + "learning_rate": 3.179221706533381e-05, + "logits/chosen": 20.803436279296875, + "logits/rejected": 20.647022247314453, + "logps/chosen": -42.571128845214844, + "logps/rejected": -34.00419235229492, + "loss": 0.5064, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.104473352432251, + "rewards/margins": 0.35474246740341187, + "rewards/rejected": -1.4592156410217285, + "step": 1020 + }, + { + "epoch": 0.367725812209925, + "grad_norm": 5.063046455383301, + "learning_rate": 3.161370938950375e-05, + "logits/chosen": 23.382953643798828, + "logits/rejected": 23.356374740600586, + "logps/chosen": -35.55125045776367, + "logps/rejected": -40.64656066894531, + "loss": 0.5387, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9775670766830444, + "rewards/margins": 0.6872422099113464, + "rewards/rejected": -1.664809226989746, + "step": 1030 + }, + { + "epoch": 0.37129596572652623, + "grad_norm": 7.303701877593994, + "learning_rate": 3.143520171367369e-05, + "logits/chosen": 24.133228302001953, + "logits/rejected": 24.389375686645508, + "logps/chosen": -50.63660430908203, + "logps/rejected": -54.03656005859375, + "loss": 0.4985, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.38200637698173523, + "rewards/margins": 1.240886926651001, + "rewards/rejected": -1.6228930950164795, + "step": 1040 + }, + { + "epoch": 0.37486611924312746, + "grad_norm": 6.75851583480835, + "learning_rate": 3.125669403784363e-05, + "logits/chosen": 19.592905044555664, + "logits/rejected": 19.829776763916016, + "logps/chosen": -59.42816162109375, + "logps/rejected": -58.9128303527832, + "loss": 0.4953, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.082971215248108, + "rewards/margins": 0.658551037311554, + "rewards/rejected": -1.7415224313735962, + "step": 1050 + }, + { + "epoch": 0.3784362727597287, + "grad_norm": 8.15499210357666, + "learning_rate": 3.1078186362013566e-05, + "logits/chosen": 25.108243942260742, + "logits/rejected": 25.314802169799805, + "logps/chosen": -44.924949645996094, + "logps/rejected": -37.734554290771484, + "loss": 0.4964, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.3476589620113373, + "rewards/margins": 1.265342354774475, + "rewards/rejected": -1.6130014657974243, + "step": 1060 + }, + { + "epoch": 0.38200642627632986, + "grad_norm": 14.444954872131348, + "learning_rate": 3.089967868618351e-05, + "logits/chosen": 24.31263542175293, + "logits/rejected": 24.08812141418457, + "logps/chosen": -42.63981246948242, + "logps/rejected": -52.27241897583008, + "loss": 0.5519, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.5805258750915527, + "rewards/margins": 0.8533035516738892, + "rewards/rejected": -1.4338295459747314, + "step": 1070 + }, + { + "epoch": 0.3855765797929311, + "grad_norm": 7.73518180847168, + "learning_rate": 3.072117101035345e-05, + "logits/chosen": 25.645715713500977, + "logits/rejected": 25.8345947265625, + "logps/chosen": -39.769630432128906, + "logps/rejected": -35.580650329589844, + "loss": 0.4351, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.4297657907009125, + "rewards/margins": 1.0592445135116577, + "rewards/rejected": -1.4890105724334717, + "step": 1080 + }, + { + "epoch": 0.3891467333095323, + "grad_norm": 6.938913345336914, + "learning_rate": 3.0542663334523384e-05, + "logits/chosen": 21.569717407226562, + "logits/rejected": 21.530803680419922, + "logps/chosen": -42.12656021118164, + "logps/rejected": -51.796173095703125, + "loss": 0.5753, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.23102021217346191, + "rewards/margins": 1.1891123056411743, + "rewards/rejected": -1.4201325178146362, + "step": 1090 + }, + { + "epoch": 0.39271688682613354, + "grad_norm": 14.953814506530762, + "learning_rate": 3.0364155658693328e-05, + "logits/chosen": 23.64889144897461, + "logits/rejected": 24.02678680419922, + "logps/chosen": -32.63819122314453, + "logps/rejected": -34.16315841674805, + "loss": 0.5176, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6592009663581848, + "rewards/margins": 1.2730789184570312, + "rewards/rejected": -1.9322798252105713, + "step": 1100 + }, + { + "epoch": 0.39271688682613354, + "eval_logits/chosen": 22.99826431274414, + "eval_logits/rejected": 23.05336570739746, + "eval_logps/chosen": -45.58966064453125, + "eval_logps/rejected": -47.14959716796875, + "eval_loss": 0.468368798494339, + "eval_rewards/accuracies": 0.8055555820465088, + "eval_rewards/chosen": -0.3949766457080841, + "eval_rewards/margins": 1.116257905960083, + "eval_rewards/rejected": -1.5112345218658447, + "eval_runtime": 97.8365, + "eval_samples_per_second": 14.637, + "eval_steps_per_second": 0.92, + "step": 1100 + }, + { + "epoch": 0.39628704034273476, + "grad_norm": 8.018348693847656, + "learning_rate": 3.0185647982863262e-05, + "logits/chosen": 21.868206024169922, + "logits/rejected": 21.934436798095703, + "logps/chosen": -12.12812614440918, + "logps/rejected": -39.364479064941406, + "loss": 0.6244, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.7536962032318115, + "rewards/margins": 0.8047281503677368, + "rewards/rejected": -1.5584242343902588, + "step": 1110 + }, + { + "epoch": 0.39985719385933594, + "grad_norm": 14.252638816833496, + "learning_rate": 3.0007140307033206e-05, + "logits/chosen": 21.122318267822266, + "logits/rejected": 21.303096771240234, + "logps/chosen": -25.239931106567383, + "logps/rejected": -44.85738754272461, + "loss": 0.5106, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9396845102310181, + "rewards/margins": 0.4221257269382477, + "rewards/rejected": -1.3618100881576538, + "step": 1120 + }, + { + "epoch": 0.40342734737593716, + "grad_norm": 10.65608024597168, + "learning_rate": 2.982863263120314e-05, + "logits/chosen": 27.12192153930664, + "logits/rejected": 27.36038589477539, + "logps/chosen": -30.87465476989746, + "logps/rejected": -33.55220031738281, + "loss": 0.5345, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.76970374584198, + "rewards/margins": 0.6138733625411987, + "rewards/rejected": -1.3835771083831787, + "step": 1130 + }, + { + "epoch": 0.4069975008925384, + "grad_norm": 8.322869300842285, + "learning_rate": 2.965012495537308e-05, + "logits/chosen": 21.752477645874023, + "logits/rejected": 21.980819702148438, + "logps/chosen": -45.920928955078125, + "logps/rejected": -35.359962463378906, + "loss": 0.4077, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -1.1174064874649048, + "rewards/margins": 0.1437036395072937, + "rewards/rejected": -1.2611101865768433, + "step": 1140 + }, + { + "epoch": 0.4105676544091396, + "grad_norm": 7.159435272216797, + "learning_rate": 2.9471617279543025e-05, + "logits/chosen": 25.57335662841797, + "logits/rejected": 25.599693298339844, + "logps/chosen": -35.72122573852539, + "logps/rejected": -55.0676383972168, + "loss": 0.4664, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.37138763070106506, + "rewards/margins": 1.5116512775421143, + "rewards/rejected": -1.883039116859436, + "step": 1150 + }, + { + "epoch": 0.4141378079257408, + "grad_norm": 6.634488582611084, + "learning_rate": 2.929310960371296e-05, + "logits/chosen": 20.41391944885254, + "logits/rejected": 20.523845672607422, + "logps/chosen": -42.240272521972656, + "logps/rejected": -59.96808624267578, + "loss": 0.3929, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6876804828643799, + "rewards/margins": 1.695648431777954, + "rewards/rejected": -2.383328914642334, + "step": 1160 + }, + { + "epoch": 0.417707961442342, + "grad_norm": 6.726844787597656, + "learning_rate": 2.91146019278829e-05, + "logits/chosen": 25.41510009765625, + "logits/rejected": 25.722497940063477, + "logps/chosen": -66.07826232910156, + "logps/rejected": -49.482093811035156, + "loss": 0.493, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.5786004066467285, + "rewards/margins": 1.6916053295135498, + "rewards/rejected": -2.2702059745788574, + "step": 1170 + }, + { + "epoch": 0.42127811495894324, + "grad_norm": 5.256776332855225, + "learning_rate": 2.8936094252052844e-05, + "logits/chosen": 24.357757568359375, + "logits/rejected": 24.397367477416992, + "logps/chosen": -45.10001754760742, + "logps/rejected": -79.18079376220703, + "loss": 0.4627, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.1261012703180313, + "rewards/margins": 1.3214435577392578, + "rewards/rejected": -1.447544813156128, + "step": 1180 + }, + { + "epoch": 0.42484826847554447, + "grad_norm": 23.727317810058594, + "learning_rate": 2.8757586576222777e-05, + "logits/chosen": 26.00099754333496, + "logits/rejected": 25.485157012939453, + "logps/chosen": -36.56196594238281, + "logps/rejected": -52.7928581237793, + "loss": 0.5167, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.9087120294570923, + "rewards/margins": 1.3440439701080322, + "rewards/rejected": -2.252755880355835, + "step": 1190 + }, + { + "epoch": 0.42841842199214564, + "grad_norm": 7.284200191497803, + "learning_rate": 2.8579078900392718e-05, + "logits/chosen": 27.846277236938477, + "logits/rejected": 28.00874900817871, + "logps/chosen": -59.95824432373047, + "logps/rejected": -73.82151794433594, + "loss": 0.4865, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.411950021982193, + "rewards/margins": 1.2028063535690308, + "rewards/rejected": -1.6147563457489014, + "step": 1200 + }, + { + "epoch": 0.42841842199214564, + "eval_logits/chosen": 22.84278106689453, + "eval_logits/rejected": 22.89705467224121, + "eval_logps/chosen": -46.42566680908203, + "eval_logps/rejected": -48.32484817504883, + "eval_loss": 0.46235060691833496, + "eval_rewards/accuracies": 0.8222222328186035, + "eval_rewards/chosen": -0.4785774350166321, + "eval_rewards/margins": 1.1501818895339966, + "eval_rewards/rejected": -1.6287592649459839, + "eval_runtime": 97.7745, + "eval_samples_per_second": 14.646, + "eval_steps_per_second": 0.92, + "step": 1200 + }, + { + "epoch": 0.43198857550874686, + "grad_norm": 10.374000549316406, + "learning_rate": 2.8400571224562655e-05, + "logits/chosen": 27.34882164001465, + "logits/rejected": 26.882160186767578, + "logps/chosen": -17.03693199157715, + "logps/rejected": -37.147464752197266, + "loss": 0.5116, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.975134015083313, + "rewards/margins": 0.5972901582717896, + "rewards/rejected": -1.5724241733551025, + "step": 1210 + }, + { + "epoch": 0.4355587290253481, + "grad_norm": 8.28000259399414, + "learning_rate": 2.8222063548732596e-05, + "logits/chosen": 24.054485321044922, + "logits/rejected": 24.115304946899414, + "logps/chosen": -40.294517517089844, + "logps/rejected": -29.917953491210938, + "loss": 0.4612, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.7177508473396301, + "rewards/margins": 0.1164645105600357, + "rewards/rejected": -0.8342153429985046, + "step": 1220 + }, + { + "epoch": 0.4391288825419493, + "grad_norm": 8.388548851013184, + "learning_rate": 2.8043555872902537e-05, + "logits/chosen": 25.40005111694336, + "logits/rejected": 25.363679885864258, + "logps/chosen": -33.07610321044922, + "logps/rejected": -46.69367218017578, + "loss": 0.5705, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6370976567268372, + "rewards/margins": 0.2812044620513916, + "rewards/rejected": -0.9183019399642944, + "step": 1230 + }, + { + "epoch": 0.44269903605855054, + "grad_norm": 10.984318733215332, + "learning_rate": 2.7865048197072474e-05, + "logits/chosen": 21.659027099609375, + "logits/rejected": 21.85382843017578, + "logps/chosen": -73.49362182617188, + "logps/rejected": -69.6221694946289, + "loss": 0.5531, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.2802386283874512, + "rewards/margins": 0.23997211456298828, + "rewards/rejected": -1.520210862159729, + "step": 1240 + }, + { + "epoch": 0.4462691895751517, + "grad_norm": 6.512259006500244, + "learning_rate": 2.7686540521242415e-05, + "logits/chosen": 25.227018356323242, + "logits/rejected": 24.743755340576172, + "logps/chosen": -47.708919525146484, + "logps/rejected": -72.66358947753906, + "loss": 0.5644, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -1.2783235311508179, + "rewards/margins": 0.3692890405654907, + "rewards/rejected": -1.6476128101348877, + "step": 1250 + }, + { + "epoch": 0.44983934309175294, + "grad_norm": 13.5468168258667, + "learning_rate": 2.7508032845412352e-05, + "logits/chosen": 22.638336181640625, + "logits/rejected": 22.76083755493164, + "logps/chosen": -33.97126007080078, + "logps/rejected": -34.66718673706055, + "loss": 0.486, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6014578342437744, + "rewards/margins": 1.0001842975616455, + "rewards/rejected": -1.6016420125961304, + "step": 1260 + }, + { + "epoch": 0.45340949660835417, + "grad_norm": 10.532015800476074, + "learning_rate": 2.7329525169582293e-05, + "logits/chosen": 20.29448699951172, + "logits/rejected": 20.10554313659668, + "logps/chosen": -36.69490051269531, + "logps/rejected": -56.0795783996582, + "loss": 0.5096, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.4149419665336609, + "rewards/margins": 0.9424688220024109, + "rewards/rejected": -1.3574106693267822, + "step": 1270 + }, + { + "epoch": 0.4569796501249554, + "grad_norm": 5.038132190704346, + "learning_rate": 2.7151017493752233e-05, + "logits/chosen": 25.44969367980957, + "logits/rejected": 25.12093734741211, + "logps/chosen": -45.38970184326172, + "logps/rejected": -48.31528091430664, + "loss": 0.4794, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.48757249116897583, + "rewards/margins": 0.5879091024398804, + "rewards/rejected": -1.075481653213501, + "step": 1280 + }, + { + "epoch": 0.46054980364155657, + "grad_norm": 14.402963638305664, + "learning_rate": 2.697250981792217e-05, + "logits/chosen": 24.284452438354492, + "logits/rejected": 23.935733795166016, + "logps/chosen": -44.071624755859375, + "logps/rejected": -44.24872589111328, + "loss": 0.5999, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -1.4019516706466675, + "rewards/margins": 0.15506015717983246, + "rewards/rejected": -1.5570119619369507, + "step": 1290 + }, + { + "epoch": 0.4641199571581578, + "grad_norm": 7.078202247619629, + "learning_rate": 2.679400214209211e-05, + "logits/chosen": 20.088533401489258, + "logits/rejected": 20.420536041259766, + "logps/chosen": -50.00440216064453, + "logps/rejected": -56.248382568359375, + "loss": 0.4642, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.6917403936386108, + "rewards/margins": 1.1690679788589478, + "rewards/rejected": -1.8608081340789795, + "step": 1300 + }, + { + "epoch": 0.4641199571581578, + "eval_logits/chosen": 21.782039642333984, + "eval_logits/rejected": 21.82331085205078, + "eval_logps/chosen": -46.87673568725586, + "eval_logps/rejected": -49.23981475830078, + "eval_loss": 0.4559466540813446, + "eval_rewards/accuracies": 0.8388888835906982, + "eval_rewards/chosen": -0.523684024810791, + "eval_rewards/margins": 1.196572184562683, + "eval_rewards/rejected": -1.7202562093734741, + "eval_runtime": 97.7244, + "eval_samples_per_second": 14.653, + "eval_steps_per_second": 0.921, + "step": 1300 + }, + { + "epoch": 0.467690110674759, + "grad_norm": 10.393937110900879, + "learning_rate": 2.6615494466262052e-05, + "logits/chosen": 20.803997039794922, + "logits/rejected": 20.7797794342041, + "logps/chosen": -44.06121826171875, + "logps/rejected": -47.28874588012695, + "loss": 0.5374, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.47202056646347046, + "rewards/margins": 0.9966621398925781, + "rewards/rejected": -1.4686826467514038, + "step": 1310 + }, + { + "epoch": 0.47126026419136025, + "grad_norm": 6.863224983215332, + "learning_rate": 2.643698679043199e-05, + "logits/chosen": 19.93913459777832, + "logits/rejected": 20.202621459960938, + "logps/chosen": -37.7781867980957, + "logps/rejected": -39.12306594848633, + "loss": 0.5426, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.0681989192962646, + "rewards/margins": 0.03398240730166435, + "rewards/rejected": -1.1021811962127686, + "step": 1320 + }, + { + "epoch": 0.4748304177079614, + "grad_norm": 9.727211952209473, + "learning_rate": 2.625847911460193e-05, + "logits/chosen": 19.134681701660156, + "logits/rejected": 19.11941146850586, + "logps/chosen": -57.157989501953125, + "logps/rejected": -72.72858428955078, + "loss": 0.507, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9592474102973938, + "rewards/margins": 1.3726739883422852, + "rewards/rejected": -2.331921339035034, + "step": 1330 + }, + { + "epoch": 0.47840057122456264, + "grad_norm": 9.726844787597656, + "learning_rate": 2.6079971438771867e-05, + "logits/chosen": 22.685604095458984, + "logits/rejected": 23.21428108215332, + "logps/chosen": -57.5455436706543, + "logps/rejected": -83.27085876464844, + "loss": 0.379, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.020816683769226, + "rewards/margins": 1.3953444957733154, + "rewards/rejected": -2.416161298751831, + "step": 1340 + }, + { + "epoch": 0.48197072474116387, + "grad_norm": 9.248942375183105, + "learning_rate": 2.5901463762941808e-05, + "logits/chosen": 19.568103790283203, + "logits/rejected": 19.459789276123047, + "logps/chosen": -22.311328887939453, + "logps/rejected": -22.42624282836914, + "loss": 0.4095, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.12884020805358887, + "rewards/margins": 1.4438714981079102, + "rewards/rejected": -1.572711706161499, + "step": 1350 + }, + { + "epoch": 0.4855408782577651, + "grad_norm": 9.093886375427246, + "learning_rate": 2.572295608711175e-05, + "logits/chosen": 19.389034271240234, + "logits/rejected": 19.35251235961914, + "logps/chosen": -33.7999382019043, + "logps/rejected": -42.553993225097656, + "loss": 0.4346, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.5412906408309937, + "rewards/margins": 0.463665634393692, + "rewards/rejected": -1.0049562454223633, + "step": 1360 + }, + { + "epoch": 0.4891110317743663, + "grad_norm": 11.408600807189941, + "learning_rate": 2.5544448411281686e-05, + "logits/chosen": 19.231670379638672, + "logits/rejected": 19.629993438720703, + "logps/chosen": -31.9910831451416, + "logps/rejected": -59.96710205078125, + "loss": 0.6003, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.4423964023590088, + "rewards/margins": 1.090103268623352, + "rewards/rejected": -1.5324996709823608, + "step": 1370 + }, + { + "epoch": 0.4926811852909675, + "grad_norm": 5.170421123504639, + "learning_rate": 2.5365940735451627e-05, + "logits/chosen": 27.88661766052246, + "logits/rejected": 27.40877914428711, + "logps/chosen": -45.87278366088867, + "logps/rejected": -45.47796630859375, + "loss": 0.4592, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.8744171261787415, + "rewards/margins": 0.6523105502128601, + "rewards/rejected": -1.5267279148101807, + "step": 1380 + }, + { + "epoch": 0.4962513388075687, + "grad_norm": 5.33081579208374, + "learning_rate": 2.5187433059621564e-05, + "logits/chosen": 21.309518814086914, + "logits/rejected": 21.40631866455078, + "logps/chosen": -36.923614501953125, + "logps/rejected": -56.65825271606445, + "loss": 0.5338, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.4122052192687988, + "rewards/margins": 0.27405181527137756, + "rewards/rejected": -1.686257004737854, + "step": 1390 + }, + { + "epoch": 0.49982149232416995, + "grad_norm": 10.743315696716309, + "learning_rate": 2.5008925383791505e-05, + "logits/chosen": 24.20417594909668, + "logits/rejected": 24.082937240600586, + "logps/chosen": -33.370670318603516, + "logps/rejected": -40.95515441894531, + "loss": 0.4764, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.0275386571884155, + "rewards/margins": 0.8042508959770203, + "rewards/rejected": -1.8317893743515015, + "step": 1400 + }, + { + "epoch": 0.49982149232416995, + "eval_logits/chosen": 20.522747039794922, + "eval_logits/rejected": 20.564208984375, + "eval_logps/chosen": -48.346073150634766, + "eval_logps/rejected": -51.10609817504883, + "eval_loss": 0.4600922167301178, + "eval_rewards/accuracies": 0.800000011920929, + "eval_rewards/chosen": -0.6706183552742004, + "eval_rewards/margins": 1.2362663745880127, + "eval_rewards/rejected": -1.906884789466858, + "eval_runtime": 99.1997, + "eval_samples_per_second": 14.436, + "eval_steps_per_second": 0.907, + "step": 1400 + }, + { + "epoch": 0.5033916458407711, + "grad_norm": 7.706882476806641, + "learning_rate": 2.4830417707961442e-05, + "logits/chosen": 18.212251663208008, + "logits/rejected": 17.935840606689453, + "logps/chosen": -55.223960876464844, + "logps/rejected": -50.36370086669922, + "loss": 0.4297, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.1679266691207886, + "rewards/margins": 1.6370041370391846, + "rewards/rejected": -2.8049309253692627, + "step": 1410 + }, + { + "epoch": 0.5069617993573724, + "grad_norm": 9.615565299987793, + "learning_rate": 2.4651910032131383e-05, + "logits/chosen": 22.241762161254883, + "logits/rejected": 22.47495460510254, + "logps/chosen": -67.49073791503906, + "logps/rejected": -73.30606079101562, + "loss": 0.4645, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -1.1100456714630127, + "rewards/margins": 1.8767004013061523, + "rewards/rejected": -2.986745834350586, + "step": 1420 + }, + { + "epoch": 0.5105319528739736, + "grad_norm": 5.209810733795166, + "learning_rate": 2.4473402356301323e-05, + "logits/chosen": 22.897598266601562, + "logits/rejected": 22.638469696044922, + "logps/chosen": -53.96964645385742, + "logps/rejected": -58.29240036010742, + "loss": 0.4386, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6232610940933228, + "rewards/margins": 0.9309207201004028, + "rewards/rejected": -1.554181694984436, + "step": 1430 + }, + { + "epoch": 0.5141021063905747, + "grad_norm": 8.026928901672363, + "learning_rate": 2.429489468047126e-05, + "logits/chosen": 22.4190731048584, + "logits/rejected": 22.359643936157227, + "logps/chosen": -48.24509048461914, + "logps/rejected": -47.77333450317383, + "loss": 0.4485, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.1848137378692627, + "rewards/margins": 0.29461732506752014, + "rewards/rejected": -1.479430913925171, + "step": 1440 + }, + { + "epoch": 0.517672259907176, + "grad_norm": 10.013033866882324, + "learning_rate": 2.41163870046412e-05, + "logits/chosen": 22.765010833740234, + "logits/rejected": 22.88141441345215, + "logps/chosen": -54.80376052856445, + "logps/rejected": -59.78020477294922, + "loss": 0.5059, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.1334106922149658, + "rewards/margins": 0.6534554362297058, + "rewards/rejected": -1.7868661880493164, + "step": 1450 + }, + { + "epoch": 0.5212424134237772, + "grad_norm": 9.498845100402832, + "learning_rate": 2.3937879328811142e-05, + "logits/chosen": 22.87884521484375, + "logits/rejected": 22.664630889892578, + "logps/chosen": -38.91038131713867, + "logps/rejected": -40.58343505859375, + "loss": 0.4454, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.4328577518463135, + "rewards/margins": 1.4672273397445679, + "rewards/rejected": -1.9000848531723022, + "step": 1460 + }, + { + "epoch": 0.5248125669403785, + "grad_norm": 9.389070510864258, + "learning_rate": 2.375937165298108e-05, + "logits/chosen": 23.82521629333496, + "logits/rejected": 24.009113311767578, + "logps/chosen": -60.59919357299805, + "logps/rejected": -68.24311828613281, + "loss": 0.4305, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.1264002323150635, + "rewards/margins": 1.3599331378936768, + "rewards/rejected": -2.4863333702087402, + "step": 1470 + }, + { + "epoch": 0.5283827204569796, + "grad_norm": 11.059606552124023, + "learning_rate": 2.358086397715102e-05, + "logits/chosen": 25.180397033691406, + "logits/rejected": 25.406417846679688, + "logps/chosen": -53.621864318847656, + "logps/rejected": -58.304664611816406, + "loss": 0.471, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.3794161081314087, + "rewards/margins": 0.9802428483963013, + "rewards/rejected": -2.359658718109131, + "step": 1480 + }, + { + "epoch": 0.5319528739735808, + "grad_norm": 19.903459548950195, + "learning_rate": 2.3402356301320957e-05, + "logits/chosen": 23.71100616455078, + "logits/rejected": 23.361339569091797, + "logps/chosen": -51.43365478515625, + "logps/rejected": -51.136474609375, + "loss": 0.4899, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7299798727035522, + "rewards/margins": 1.1167805194854736, + "rewards/rejected": -1.8467603921890259, + "step": 1490 + }, + { + "epoch": 0.5355230274901821, + "grad_norm": 9.738292694091797, + "learning_rate": 2.3223848625490895e-05, + "logits/chosen": 23.410078048706055, + "logits/rejected": 23.206371307373047, + "logps/chosen": -47.294952392578125, + "logps/rejected": -54.903099060058594, + "loss": 0.4734, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9249691963195801, + "rewards/margins": 0.9751961827278137, + "rewards/rejected": -1.9001652002334595, + "step": 1500 + }, + { + "epoch": 0.5355230274901821, + "eval_logits/chosen": 20.92539405822754, + "eval_logits/rejected": 20.960508346557617, + "eval_logps/chosen": -48.110416412353516, + "eval_logps/rejected": -51.09627151489258, + "eval_loss": 0.4577901065349579, + "eval_rewards/accuracies": 0.800000011920929, + "eval_rewards/chosen": -0.6470518112182617, + "eval_rewards/margins": 1.2588502168655396, + "eval_rewards/rejected": -1.9059022665023804, + "eval_runtime": 97.9322, + "eval_samples_per_second": 14.622, + "eval_steps_per_second": 0.919, + "step": 1500 + }, + { + "epoch": 0.5390931810067833, + "grad_norm": 8.651128768920898, + "learning_rate": 2.304534094966084e-05, + "logits/chosen": 22.864696502685547, + "logits/rejected": 22.489166259765625, + "logps/chosen": -24.272085189819336, + "logps/rejected": -25.032752990722656, + "loss": 0.5041, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.06202169135212898, + "rewards/margins": 1.4595599174499512, + "rewards/rejected": -1.5215816497802734, + "step": 1510 + }, + { + "epoch": 0.5426633345233846, + "grad_norm": 13.196417808532715, + "learning_rate": 2.2866833273830776e-05, + "logits/chosen": 23.409244537353516, + "logits/rejected": 23.42110824584961, + "logps/chosen": -70.71308898925781, + "logps/rejected": -65.17359161376953, + "loss": 0.4541, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.825936496257782, + "rewards/margins": 1.5356091260910034, + "rewards/rejected": -2.3615458011627197, + "step": 1520 + }, + { + "epoch": 0.5462334880399857, + "grad_norm": 13.838322639465332, + "learning_rate": 2.2688325598000713e-05, + "logits/chosen": 19.617794036865234, + "logits/rejected": 19.378015518188477, + "logps/chosen": -63.78108596801758, + "logps/rejected": -57.444053649902344, + "loss": 0.5129, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.7819554805755615, + "rewards/margins": 1.4755408763885498, + "rewards/rejected": -2.2574961185455322, + "step": 1530 + }, + { + "epoch": 0.5498036415565869, + "grad_norm": 3.351306915283203, + "learning_rate": 2.2509817922170654e-05, + "logits/chosen": 18.362462997436523, + "logits/rejected": 18.116527557373047, + "logps/chosen": -61.28546905517578, + "logps/rejected": -71.31098937988281, + "loss": 0.667, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7423151135444641, + "rewards/margins": 1.1754957437515259, + "rewards/rejected": -1.9178107976913452, + "step": 1540 + }, + { + "epoch": 0.5533737950731882, + "grad_norm": 10.512808799743652, + "learning_rate": 2.2331310246340595e-05, + "logits/chosen": 23.116546630859375, + "logits/rejected": 23.27178955078125, + "logps/chosen": -71.55606079101562, + "logps/rejected": -59.09543991088867, + "loss": 0.5858, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.0912095308303833, + "rewards/margins": 1.2087962627410889, + "rewards/rejected": -2.3000059127807617, + "step": 1550 + }, + { + "epoch": 0.5569439485897894, + "grad_norm": 6.164778709411621, + "learning_rate": 2.2152802570510532e-05, + "logits/chosen": 22.27012062072754, + "logits/rejected": 22.54111671447754, + "logps/chosen": -50.849666595458984, + "logps/rejected": -51.00347900390625, + "loss": 0.4023, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.8672236204147339, + "rewards/margins": 0.6719032526016235, + "rewards/rejected": -1.539126992225647, + "step": 1560 + }, + { + "epoch": 0.5605141021063905, + "grad_norm": 8.524353981018066, + "learning_rate": 2.1974294894680473e-05, + "logits/chosen": 24.240970611572266, + "logits/rejected": 24.447324752807617, + "logps/chosen": -50.92961120605469, + "logps/rejected": -25.607860565185547, + "loss": 0.5566, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.0541558265686035, + "rewards/margins": 0.1918071210384369, + "rewards/rejected": -1.2459629774093628, + "step": 1570 + }, + { + "epoch": 0.5640842556229918, + "grad_norm": 3.2097225189208984, + "learning_rate": 2.179578721885041e-05, + "logits/chosen": 21.733001708984375, + "logits/rejected": 21.849323272705078, + "logps/chosen": -59.861595153808594, + "logps/rejected": -74.30784606933594, + "loss": 0.4727, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.154857873916626, + "rewards/margins": 1.554736852645874, + "rewards/rejected": -2.709594249725342, + "step": 1580 + }, + { + "epoch": 0.567654409139593, + "grad_norm": 10.112348556518555, + "learning_rate": 2.1617279543020354e-05, + "logits/chosen": 22.288734436035156, + "logits/rejected": 22.532554626464844, + "logps/chosen": -49.292564392089844, + "logps/rejected": -46.22431182861328, + "loss": 0.3915, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.7656620144844055, + "rewards/margins": 0.9457171559333801, + "rewards/rejected": -1.7113794088363647, + "step": 1590 + }, + { + "epoch": 0.5712245626561943, + "grad_norm": 7.5602288246154785, + "learning_rate": 2.143877186719029e-05, + "logits/chosen": 16.03813934326172, + "logits/rejected": 16.125709533691406, + "logps/chosen": -48.10554504394531, + "logps/rejected": -57.5333137512207, + "loss": 0.4459, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9440638422966003, + "rewards/margins": 0.8066110610961914, + "rewards/rejected": -1.750674843788147, + "step": 1600 + }, + { + "epoch": 0.5712245626561943, + "eval_logits/chosen": 21.25445556640625, + "eval_logits/rejected": 21.29209327697754, + "eval_logps/chosen": -46.128841400146484, + "eval_logps/rejected": -50.25452423095703, + "eval_loss": 0.4501490294933319, + "eval_rewards/accuracies": 0.8166666626930237, + "eval_rewards/chosen": -0.4488947093486786, + "eval_rewards/margins": 1.372832179069519, + "eval_rewards/rejected": -1.8217267990112305, + "eval_runtime": 97.5045, + "eval_samples_per_second": 14.687, + "eval_steps_per_second": 0.923, + "step": 1600 + }, + { + "epoch": 0.5747947161727954, + "grad_norm": 21.815628051757812, + "learning_rate": 2.126026419136023e-05, + "logits/chosen": 22.183074951171875, + "logits/rejected": 23.01669692993164, + "logps/chosen": -25.743982315063477, + "logps/rejected": -32.8138427734375, + "loss": 0.4382, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7791844606399536, + "rewards/margins": 1.4727532863616943, + "rewards/rejected": -2.2519376277923584, + "step": 1610 + }, + { + "epoch": 0.5783648696893966, + "grad_norm": 9.609827041625977, + "learning_rate": 2.108175651553017e-05, + "logits/chosen": 21.618915557861328, + "logits/rejected": 21.331867218017578, + "logps/chosen": -36.85995864868164, + "logps/rejected": -43.229026794433594, + "loss": 0.7286, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.826258659362793, + "rewards/margins": 1.1748714447021484, + "rewards/rejected": -2.0011298656463623, + "step": 1620 + }, + { + "epoch": 0.5819350232059979, + "grad_norm": 4.5630059242248535, + "learning_rate": 2.0903248839700106e-05, + "logits/chosen": 19.84273910522461, + "logits/rejected": 19.884912490844727, + "logps/chosen": -30.239971160888672, + "logps/rejected": -48.68877029418945, + "loss": 0.4667, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.15234649181365967, + "rewards/margins": 1.829787015914917, + "rewards/rejected": -1.9821338653564453, + "step": 1630 + }, + { + "epoch": 0.585505176722599, + "grad_norm": 10.104663848876953, + "learning_rate": 2.0724741163870047e-05, + "logits/chosen": 22.992443084716797, + "logits/rejected": 23.00868034362793, + "logps/chosen": -63.122642517089844, + "logps/rejected": -61.594627380371094, + "loss": 0.4673, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.868377685546875, + "rewards/margins": 1.27804696559906, + "rewards/rejected": -2.1464245319366455, + "step": 1640 + }, + { + "epoch": 0.5890753302392003, + "grad_norm": 15.089373588562012, + "learning_rate": 2.0546233488039988e-05, + "logits/chosen": 25.846508026123047, + "logits/rejected": 25.832605361938477, + "logps/chosen": -36.8404655456543, + "logps/rejected": -45.35338592529297, + "loss": 0.4712, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.2539387941360474, + "rewards/margins": -0.1804993450641632, + "rewards/rejected": -1.0734394788742065, + "step": 1650 + }, + { + "epoch": 0.5926454837558015, + "grad_norm": 8.563141822814941, + "learning_rate": 2.0367725812209925e-05, + "logits/chosen": 25.35030746459961, + "logits/rejected": 25.004871368408203, + "logps/chosen": -14.423406600952148, + "logps/rejected": -27.992664337158203, + "loss": 0.538, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.5140976309776306, + "rewards/margins": 0.7111214995384216, + "rewards/rejected": -1.2252190113067627, + "step": 1660 + }, + { + "epoch": 0.5962156372724027, + "grad_norm": 10.54287338256836, + "learning_rate": 2.0189218136379866e-05, + "logits/chosen": 23.3927001953125, + "logits/rejected": 23.853105545043945, + "logps/chosen": -48.61920928955078, + "logps/rejected": -59.92339324951172, + "loss": 0.6793, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.8948885202407837, + "rewards/margins": 1.2263555526733398, + "rewards/rejected": -2.121243953704834, + "step": 1670 + }, + { + "epoch": 0.599785790789004, + "grad_norm": 10.524778366088867, + "learning_rate": 2.0010710460549806e-05, + "logits/chosen": 19.91085433959961, + "logits/rejected": 19.805999755859375, + "logps/chosen": -37.84980010986328, + "logps/rejected": -42.93219757080078, + "loss": 0.4365, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.022962704300880432, + "rewards/margins": 1.8917738199234009, + "rewards/rejected": -1.868811011314392, + "step": 1680 + }, + { + "epoch": 0.6033559443056051, + "grad_norm": 14.300918579101562, + "learning_rate": 1.9832202784719744e-05, + "logits/chosen": 18.957054138183594, + "logits/rejected": 19.1351318359375, + "logps/chosen": -44.51586151123047, + "logps/rejected": -51.03856658935547, + "loss": 0.6677, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3876987397670746, + "rewards/margins": 1.393635869026184, + "rewards/rejected": -1.7813348770141602, + "step": 1690 + }, + { + "epoch": 0.6069260978222063, + "grad_norm": 10.632288932800293, + "learning_rate": 1.9653695108889684e-05, + "logits/chosen": 23.2322998046875, + "logits/rejected": 22.36001205444336, + "logps/chosen": -33.04428482055664, + "logps/rejected": -44.446144104003906, + "loss": 0.5128, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.9377388954162598, + "rewards/margins": 1.140350341796875, + "rewards/rejected": -2.0780892372131348, + "step": 1700 + }, + { + "epoch": 0.6069260978222063, + "eval_logits/chosen": 21.207813262939453, + "eval_logits/rejected": 21.23162078857422, + "eval_logps/chosen": -46.74406433105469, + "eval_logps/rejected": -49.7892951965332, + "eval_loss": 0.4637661278247833, + "eval_rewards/accuracies": 0.8222222328186035, + "eval_rewards/chosen": -0.510417103767395, + "eval_rewards/margins": 1.2647870779037476, + "eval_rewards/rejected": -1.7752044200897217, + "eval_runtime": 97.7592, + "eval_samples_per_second": 14.648, + "eval_steps_per_second": 0.921, + "step": 1700 + }, + { + "epoch": 0.6104962513388076, + "grad_norm": 9.3756742477417, + "learning_rate": 1.9475187433059622e-05, + "logits/chosen": 21.191104888916016, + "logits/rejected": 21.159591674804688, + "logps/chosen": -31.99550437927246, + "logps/rejected": -44.552982330322266, + "loss": 0.3974, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.8665735125541687, + "rewards/margins": 1.130059838294983, + "rewards/rejected": -1.9966332912445068, + "step": 1710 + }, + { + "epoch": 0.6140664048554088, + "grad_norm": 21.877330780029297, + "learning_rate": 1.929667975722956e-05, + "logits/chosen": 23.222850799560547, + "logits/rejected": 23.243497848510742, + "logps/chosen": -36.72397232055664, + "logps/rejected": -67.38380432128906, + "loss": 0.5978, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6531896591186523, + "rewards/margins": 1.578351616859436, + "rewards/rejected": -2.231541156768799, + "step": 1720 + }, + { + "epoch": 0.61763655837201, + "grad_norm": 6.913092136383057, + "learning_rate": 1.9118172081399503e-05, + "logits/chosen": 22.709789276123047, + "logits/rejected": 22.42465591430664, + "logps/chosen": -80.70012664794922, + "logps/rejected": -39.1956901550293, + "loss": 0.5559, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.349876880645752, + "rewards/margins": 0.47570380568504333, + "rewards/rejected": -1.8255808353424072, + "step": 1730 + }, + { + "epoch": 0.6212067118886112, + "grad_norm": 8.642501831054688, + "learning_rate": 1.893966440556944e-05, + "logits/chosen": 22.583799362182617, + "logits/rejected": 22.996828079223633, + "logps/chosen": -33.973365783691406, + "logps/rejected": -40.40299987792969, + "loss": 0.5488, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8000438809394836, + "rewards/margins": 1.3411608934402466, + "rewards/rejected": -2.141204595565796, + "step": 1740 + }, + { + "epoch": 0.6247768654052124, + "grad_norm": 7.104326248168945, + "learning_rate": 1.8761156729739378e-05, + "logits/chosen": 19.01543426513672, + "logits/rejected": 19.197078704833984, + "logps/chosen": -56.73799514770508, + "logps/rejected": -57.29972457885742, + "loss": 0.3864, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.8416491746902466, + "rewards/margins": 1.005480170249939, + "rewards/rejected": -1.847129464149475, + "step": 1750 + }, + { + "epoch": 0.6283470189218137, + "grad_norm": 9.371530532836914, + "learning_rate": 1.858264905390932e-05, + "logits/chosen": 23.223146438598633, + "logits/rejected": 23.122949600219727, + "logps/chosen": -45.33961868286133, + "logps/rejected": -50.840667724609375, + "loss": 0.5181, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.8626285791397095, + "rewards/margins": 0.9665843844413757, + "rewards/rejected": -1.8292129039764404, + "step": 1760 + }, + { + "epoch": 0.6319171724384148, + "grad_norm": 9.019303321838379, + "learning_rate": 1.840414137807926e-05, + "logits/chosen": 25.988510131835938, + "logits/rejected": 25.829853057861328, + "logps/chosen": -39.30087661743164, + "logps/rejected": -33.74537658691406, + "loss": 0.5334, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.3033127784729004, + "rewards/margins": 0.49060168862342834, + "rewards/rejected": -1.793914556503296, + "step": 1770 + }, + { + "epoch": 0.6354873259550161, + "grad_norm": 23.491329193115234, + "learning_rate": 1.8225633702249196e-05, + "logits/chosen": 23.851768493652344, + "logits/rejected": 24.093891143798828, + "logps/chosen": -38.49626541137695, + "logps/rejected": -29.503376007080078, + "loss": 0.5504, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6341218948364258, + "rewards/margins": 0.8659729957580566, + "rewards/rejected": -1.5000948905944824, + "step": 1780 + }, + { + "epoch": 0.6390574794716173, + "grad_norm": 11.341009140014648, + "learning_rate": 1.8047126026419137e-05, + "logits/chosen": 20.8333683013916, + "logits/rejected": 21.53514289855957, + "logps/chosen": -30.507156372070312, + "logps/rejected": -51.23711013793945, + "loss": 0.5258, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.3501494526863098, + "rewards/margins": 1.454388976097107, + "rewards/rejected": -1.8045384883880615, + "step": 1790 + }, + { + "epoch": 0.6426276329882185, + "grad_norm": 15.42497730255127, + "learning_rate": 1.7868618350589074e-05, + "logits/chosen": 23.256488800048828, + "logits/rejected": 23.68168067932129, + "logps/chosen": -23.0216007232666, + "logps/rejected": -56.100563049316406, + "loss": 0.4459, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -1.222777247428894, + "rewards/margins": 1.2910280227661133, + "rewards/rejected": -2.5138051509857178, + "step": 1800 + }, + { + "epoch": 0.6426276329882185, + "eval_logits/chosen": 21.823591232299805, + "eval_logits/rejected": 21.842632293701172, + "eval_logps/chosen": -45.74694061279297, + "eval_logps/rejected": -48.988624572753906, + "eval_loss": 0.4546748399734497, + "eval_rewards/accuracies": 0.8111110925674438, + "eval_rewards/chosen": -0.4107048809528351, + "eval_rewards/margins": 1.2844319343566895, + "eval_rewards/rejected": -1.6951369047164917, + "eval_runtime": 97.7489, + "eval_samples_per_second": 14.65, + "eval_steps_per_second": 0.921, + "step": 1800 + }, + { + "epoch": 0.6461977865048197, + "grad_norm": 5.2203288078308105, + "learning_rate": 1.769011067475902e-05, + "logits/chosen": 24.605764389038086, + "logits/rejected": 24.0284366607666, + "logps/chosen": -44.47723388671875, + "logps/rejected": -43.85586929321289, + "loss": 0.3868, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.3751835823059082, + "rewards/margins": 2.202699899673462, + "rewards/rejected": -2.57788348197937, + "step": 1810 + }, + { + "epoch": 0.6497679400214209, + "grad_norm": 15.71258544921875, + "learning_rate": 1.7511602998928956e-05, + "logits/chosen": 17.750822067260742, + "logits/rejected": 18.020795822143555, + "logps/chosen": -12.859057426452637, + "logps/rejected": -30.338369369506836, + "loss": 0.4199, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.33368760347366333, + "rewards/margins": 1.292131781578064, + "rewards/rejected": -1.625819444656372, + "step": 1820 + }, + { + "epoch": 0.6533380935380221, + "grad_norm": 4.521580219268799, + "learning_rate": 1.7333095323098893e-05, + "logits/chosen": 23.34914207458496, + "logits/rejected": 23.413654327392578, + "logps/chosen": -37.8521614074707, + "logps/rejected": -67.8910903930664, + "loss": 0.4586, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.48519086837768555, + "rewards/margins": 1.7367101907730103, + "rewards/rejected": -2.2219011783599854, + "step": 1830 + }, + { + "epoch": 0.6569082470546234, + "grad_norm": 3.1471476554870605, + "learning_rate": 1.7154587647268834e-05, + "logits/chosen": 18.366016387939453, + "logits/rejected": 18.327919006347656, + "logps/chosen": -55.4625358581543, + "logps/rejected": -69.14282989501953, + "loss": 0.5265, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9972825050354004, + "rewards/margins": 0.521903395652771, + "rewards/rejected": -1.5191859006881714, + "step": 1840 + }, + { + "epoch": 0.6604784005712245, + "grad_norm": 12.677329063415527, + "learning_rate": 1.697607997143877e-05, + "logits/chosen": 22.81624984741211, + "logits/rejected": 22.75391960144043, + "logps/chosen": -57.1756706237793, + "logps/rejected": -61.690345764160156, + "loss": 0.4095, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12716366350650787, + "rewards/margins": 1.8074487447738647, + "rewards/rejected": -1.934612512588501, + "step": 1850 + }, + { + "epoch": 0.6640485540878258, + "grad_norm": 12.960306167602539, + "learning_rate": 1.679757229560871e-05, + "logits/chosen": 23.8206844329834, + "logits/rejected": 23.760665893554688, + "logps/chosen": -41.61978530883789, + "logps/rejected": -53.76505661010742, + "loss": 0.4355, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2521887421607971, + "rewards/margins": 1.3217939138412476, + "rewards/rejected": -1.5739825963974, + "step": 1860 + }, + { + "epoch": 0.667618707604427, + "grad_norm": 16.337543487548828, + "learning_rate": 1.6619064619778652e-05, + "logits/chosen": 16.793262481689453, + "logits/rejected": 16.742605209350586, + "logps/chosen": -48.788875579833984, + "logps/rejected": -43.506622314453125, + "loss": 0.4937, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.416334331035614, + "rewards/margins": 1.9437252283096313, + "rewards/rejected": -2.3600597381591797, + "step": 1870 + }, + { + "epoch": 0.6711888611210282, + "grad_norm": 10.352446556091309, + "learning_rate": 1.644055694394859e-05, + "logits/chosen": 20.40374183654785, + "logits/rejected": 20.37020492553711, + "logps/chosen": -58.631561279296875, + "logps/rejected": -57.344520568847656, + "loss": 0.5564, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.8105586767196655, + "rewards/margins": 0.7123308777809143, + "rewards/rejected": -1.5228896141052246, + "step": 1880 + }, + { + "epoch": 0.6747590146376294, + "grad_norm": 5.631677627563477, + "learning_rate": 1.626204926811853e-05, + "logits/chosen": 23.19020652770996, + "logits/rejected": 23.061664581298828, + "logps/chosen": -61.78633499145508, + "logps/rejected": -35.33949279785156, + "loss": 0.4221, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.6975164413452148, + "rewards/margins": 1.0272176265716553, + "rewards/rejected": -1.7247339487075806, + "step": 1890 + }, + { + "epoch": 0.6783291681542306, + "grad_norm": 4.499826431274414, + "learning_rate": 1.608354159228847e-05, + "logits/chosen": 24.53340721130371, + "logits/rejected": 24.41737937927246, + "logps/chosen": -38.38304901123047, + "logps/rejected": -41.079261779785156, + "loss": 0.5325, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.7768689393997192, + "rewards/margins": 1.1220049858093262, + "rewards/rejected": -1.898874044418335, + "step": 1900 + }, + { + "epoch": 0.6783291681542306, + "eval_logits/chosen": 21.38434600830078, + "eval_logits/rejected": 21.400619506835938, + "eval_logps/chosen": -46.102115631103516, + "eval_logps/rejected": -50.66127395629883, + "eval_loss": 0.4424135386943817, + "eval_rewards/accuracies": 0.8166666626930237, + "eval_rewards/chosen": -0.4462220072746277, + "eval_rewards/margins": 1.41618013381958, + "eval_rewards/rejected": -1.862402319908142, + "eval_runtime": 97.7047, + "eval_samples_per_second": 14.656, + "eval_steps_per_second": 0.921, + "step": 1900 + }, + { + "epoch": 0.6818993216708319, + "grad_norm": 7.356407165527344, + "learning_rate": 1.5905033916458408e-05, + "logits/chosen": 22.510066986083984, + "logits/rejected": 22.577346801757812, + "logps/chosen": -35.227081298828125, + "logps/rejected": -55.93198776245117, + "loss": 0.5062, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.9009400606155396, + "rewards/margins": 1.257596731185913, + "rewards/rejected": -2.158536911010742, + "step": 1910 + }, + { + "epoch": 0.6854694751874331, + "grad_norm": 3.671786069869995, + "learning_rate": 1.572652624062835e-05, + "logits/chosen": 20.359420776367188, + "logits/rejected": 20.034189224243164, + "logps/chosen": -63.456451416015625, + "logps/rejected": -51.79132080078125, + "loss": 0.4089, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6570903658866882, + "rewards/margins": 1.6083368062973022, + "rewards/rejected": -2.265427350997925, + "step": 1920 + }, + { + "epoch": 0.6890396287040342, + "grad_norm": 10.456278800964355, + "learning_rate": 1.5548018564798286e-05, + "logits/chosen": 23.675350189208984, + "logits/rejected": 23.863216400146484, + "logps/chosen": -26.890039443969727, + "logps/rejected": -62.48893356323242, + "loss": 0.5818, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.0192630290985107, + "rewards/margins": 1.5647847652435303, + "rewards/rejected": -2.584047794342041, + "step": 1930 + }, + { + "epoch": 0.6926097822206355, + "grad_norm": 9.162726402282715, + "learning_rate": 1.5369510888968227e-05, + "logits/chosen": 23.965158462524414, + "logits/rejected": 23.98688316345215, + "logps/chosen": -21.68771743774414, + "logps/rejected": -23.776859283447266, + "loss": 0.3677, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.539600133895874, + "rewards/margins": 0.6697312593460083, + "rewards/rejected": -1.2093312740325928, + "step": 1940 + }, + { + "epoch": 0.6961799357372367, + "grad_norm": 3.7567951679229736, + "learning_rate": 1.5191003213138166e-05, + "logits/chosen": 24.331890106201172, + "logits/rejected": 24.351388931274414, + "logps/chosen": -18.292221069335938, + "logps/rejected": -28.088825225830078, + "loss": 0.4699, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.5271309614181519, + "rewards/margins": 1.1870901584625244, + "rewards/rejected": -1.7142210006713867, + "step": 1950 + }, + { + "epoch": 0.699750089253838, + "grad_norm": 10.841853141784668, + "learning_rate": 1.5012495537308105e-05, + "logits/chosen": 24.288837432861328, + "logits/rejected": 24.071971893310547, + "logps/chosen": -47.823211669921875, + "logps/rejected": -52.2962532043457, + "loss": 0.4207, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.8029332160949707, + "rewards/margins": 1.4008492231369019, + "rewards/rejected": -2.203782558441162, + "step": 1960 + }, + { + "epoch": 0.7033202427704391, + "grad_norm": 3.0264902114868164, + "learning_rate": 1.4833987861478044e-05, + "logits/chosen": 19.48323631286621, + "logits/rejected": 19.590219497680664, + "logps/chosen": -42.73374557495117, + "logps/rejected": -58.14360427856445, + "loss": 0.4225, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.6881227493286133, + "rewards/margins": 1.56888747215271, + "rewards/rejected": -2.2570104598999023, + "step": 1970 + }, + { + "epoch": 0.7068903962870403, + "grad_norm": 4.581413745880127, + "learning_rate": 1.4655480185647983e-05, + "logits/chosen": 23.510986328125, + "logits/rejected": 23.64889144897461, + "logps/chosen": -50.88288116455078, + "logps/rejected": -30.534610748291016, + "loss": 0.3636, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.262825071811676, + "rewards/margins": 1.9995416402816772, + "rewards/rejected": -2.262366771697998, + "step": 1980 + }, + { + "epoch": 0.7104605498036416, + "grad_norm": 15.385260581970215, + "learning_rate": 1.4476972509817924e-05, + "logits/chosen": 22.773696899414062, + "logits/rejected": 22.666271209716797, + "logps/chosen": -40.43771743774414, + "logps/rejected": -64.87566375732422, + "loss": 0.4299, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.787145972251892, + "rewards/margins": 1.5212252140045166, + "rewards/rejected": -3.3083713054656982, + "step": 1990 + }, + { + "epoch": 0.7140307033202428, + "grad_norm": 20.59077262878418, + "learning_rate": 1.4298464833987863e-05, + "logits/chosen": 14.725168228149414, + "logits/rejected": 15.075048446655273, + "logps/chosen": -58.4405517578125, + "logps/rejected": -52.311119079589844, + "loss": 0.5773, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.5943628549575806, + "rewards/margins": -0.12944069504737854, + "rewards/rejected": -1.4649221897125244, + "step": 2000 + }, + { + "epoch": 0.7140307033202428, + "eval_logits/chosen": 20.44503402709961, + "eval_logits/rejected": 20.467418670654297, + "eval_logps/chosen": -47.48448944091797, + "eval_logps/rejected": -52.786041259765625, + "eval_loss": 0.440356969833374, + "eval_rewards/accuracies": 0.8277778029441833, + "eval_rewards/chosen": -0.5844593644142151, + "eval_rewards/margins": 1.4904190301895142, + "eval_rewards/rejected": -2.074878454208374, + "eval_runtime": 97.8926, + "eval_samples_per_second": 14.628, + "eval_steps_per_second": 0.919, + "step": 2000 + }, + { + "epoch": 0.7176008568368439, + "grad_norm": 10.820709228515625, + "learning_rate": 1.4119957158157802e-05, + "logits/chosen": 23.97198486328125, + "logits/rejected": 23.974132537841797, + "logps/chosen": -42.35189437866211, + "logps/rejected": -57.66106414794922, + "loss": 0.5486, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.9043490290641785, + "rewards/margins": 1.2323650121688843, + "rewards/rejected": -2.136713981628418, + "step": 2010 + }, + { + "epoch": 0.7211710103534452, + "grad_norm": 4.93384313583374, + "learning_rate": 1.394144948232774e-05, + "logits/chosen": 21.256620407104492, + "logits/rejected": 20.852317810058594, + "logps/chosen": -27.885059356689453, + "logps/rejected": -38.469505310058594, + "loss": 0.4361, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.8113853335380554, + "rewards/margins": 1.1693836450576782, + "rewards/rejected": -1.9807687997817993, + "step": 2020 + }, + { + "epoch": 0.7247411638700464, + "grad_norm": 8.402702331542969, + "learning_rate": 1.3762941806497681e-05, + "logits/chosen": 22.48406982421875, + "logits/rejected": 22.659347534179688, + "logps/chosen": -30.04986572265625, + "logps/rejected": -39.668617248535156, + "loss": 0.3847, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.43164730072021484, + "rewards/margins": 1.5857521295547485, + "rewards/rejected": -2.017399549484253, + "step": 2030 + }, + { + "epoch": 0.7283113173866477, + "grad_norm": 5.305787563323975, + "learning_rate": 1.358443413066762e-05, + "logits/chosen": 24.485246658325195, + "logits/rejected": 24.59568214416504, + "logps/chosen": -61.41462326049805, + "logps/rejected": -70.96296691894531, + "loss": 0.5182, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.9171468019485474, + "rewards/margins": 0.8545171618461609, + "rewards/rejected": -1.7716642618179321, + "step": 2040 + }, + { + "epoch": 0.7318814709032488, + "grad_norm": 7.894079685211182, + "learning_rate": 1.340592645483756e-05, + "logits/chosen": 18.658212661743164, + "logits/rejected": 18.898082733154297, + "logps/chosen": -29.949609756469727, + "logps/rejected": -52.39385986328125, + "loss": 0.349, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.49497169256210327, + "rewards/margins": 1.6922725439071655, + "rewards/rejected": -2.187243938446045, + "step": 2050 + }, + { + "epoch": 0.73545162441985, + "grad_norm": 7.683061599731445, + "learning_rate": 1.3227418779007496e-05, + "logits/chosen": 21.266719818115234, + "logits/rejected": 21.34499740600586, + "logps/chosen": -37.20063018798828, + "logps/rejected": -41.06432342529297, + "loss": 0.4469, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.48325157165527344, + "rewards/margins": 0.4681263566017151, + "rewards/rejected": -0.9513779878616333, + "step": 2060 + }, + { + "epoch": 0.7390217779364513, + "grad_norm": 9.4763765335083, + "learning_rate": 1.3048911103177439e-05, + "logits/chosen": 21.261632919311523, + "logits/rejected": 21.200904846191406, + "logps/chosen": -37.98931121826172, + "logps/rejected": -50.971412658691406, + "loss": 0.4712, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.8451720476150513, + "rewards/margins": 1.3964030742645264, + "rewards/rejected": -2.241574764251709, + "step": 2070 + }, + { + "epoch": 0.7425919314530525, + "grad_norm": 12.83144474029541, + "learning_rate": 1.2870403427347378e-05, + "logits/chosen": 26.042606353759766, + "logits/rejected": 25.88700294494629, + "logps/chosen": -62.49781036376953, + "logps/rejected": -77.66305541992188, + "loss": 0.4901, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -1.1033135652542114, + "rewards/margins": 1.2711939811706543, + "rewards/rejected": -2.374507427215576, + "step": 2080 + }, + { + "epoch": 0.7461620849696537, + "grad_norm": 4.098615646362305, + "learning_rate": 1.2691895751517315e-05, + "logits/chosen": 22.199886322021484, + "logits/rejected": 22.246328353881836, + "logps/chosen": -53.25043869018555, + "logps/rejected": -53.634849548339844, + "loss": 0.447, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.8297556042671204, + "rewards/margins": 1.6126811504364014, + "rewards/rejected": -2.442436695098877, + "step": 2090 + }, + { + "epoch": 0.7497322384862549, + "grad_norm": 7.720545291900635, + "learning_rate": 1.2513388075687254e-05, + "logits/chosen": 23.562522888183594, + "logits/rejected": 23.22288703918457, + "logps/chosen": -79.41565704345703, + "logps/rejected": -45.413997650146484, + "loss": 0.4843, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.48383426666259766, + "rewards/margins": 1.4371916055679321, + "rewards/rejected": -1.9210258722305298, + "step": 2100 + }, + { + "epoch": 0.7497322384862549, + "eval_logits/chosen": 20.6312198638916, + "eval_logits/rejected": 20.657325744628906, + "eval_logps/chosen": -47.1960334777832, + "eval_logps/rejected": -52.259559631347656, + "eval_loss": 0.4409657418727875, + "eval_rewards/accuracies": 0.8333333134651184, + "eval_rewards/chosen": -0.5556142330169678, + "eval_rewards/margins": 1.4666165113449097, + "eval_rewards/rejected": -2.022230625152588, + "eval_runtime": 97.8612, + "eval_samples_per_second": 14.633, + "eval_steps_per_second": 0.92, + "step": 2100 + }, + { + "epoch": 0.7533023920028561, + "grad_norm": 11.717628479003906, + "learning_rate": 1.2334880399857195e-05, + "logits/chosen": 19.04401969909668, + "logits/rejected": 19.13692855834961, + "logps/chosen": -56.0210075378418, + "logps/rejected": -46.79364013671875, + "loss": 0.3983, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6606385707855225, + "rewards/margins": 1.5798553228378296, + "rewards/rejected": -2.2404940128326416, + "step": 2110 + }, + { + "epoch": 0.7568725455194574, + "grad_norm": 6.157966136932373, + "learning_rate": 1.2156372724027134e-05, + "logits/chosen": 17.96826171875, + "logits/rejected": 18.081592559814453, + "logps/chosen": -42.12479782104492, + "logps/rejected": -44.53673553466797, + "loss": 0.3718, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.36954426765441895, + "rewards/margins": 1.775122880935669, + "rewards/rejected": -2.144667148590088, + "step": 2120 + }, + { + "epoch": 0.7604426990360585, + "grad_norm": 3.8417954444885254, + "learning_rate": 1.1977865048197073e-05, + "logits/chosen": 20.064462661743164, + "logits/rejected": 19.88588523864746, + "logps/chosen": -44.317867279052734, + "logps/rejected": -63.211326599121094, + "loss": 0.5168, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.0525072813034058, + "rewards/margins": 1.6052157878875732, + "rewards/rejected": -2.6577229499816895, + "step": 2130 + }, + { + "epoch": 0.7640128525526597, + "grad_norm": 3.4509365558624268, + "learning_rate": 1.1799357372367012e-05, + "logits/chosen": 22.211759567260742, + "logits/rejected": 22.149486541748047, + "logps/chosen": -37.72747039794922, + "logps/rejected": -46.89754867553711, + "loss": 0.3956, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.0622680187225342, + "rewards/margins": 1.1859983205795288, + "rewards/rejected": -2.2482664585113525, + "step": 2140 + }, + { + "epoch": 0.767583006069261, + "grad_norm": 13.607682228088379, + "learning_rate": 1.1620849696536952e-05, + "logits/chosen": 23.63653564453125, + "logits/rejected": 23.572277069091797, + "logps/chosen": -41.84113311767578, + "logps/rejected": -45.45246887207031, + "loss": 0.5191, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.7334656715393066, + "rewards/margins": 1.2255051136016846, + "rewards/rejected": -1.9589707851409912, + "step": 2150 + }, + { + "epoch": 0.7711531595858622, + "grad_norm": 12.166585922241211, + "learning_rate": 1.1442342020706891e-05, + "logits/chosen": 23.18703842163086, + "logits/rejected": 23.008901596069336, + "logps/chosen": -48.22418975830078, + "logps/rejected": -39.83467102050781, + "loss": 0.4619, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.0254920721054077, + "rewards/margins": 1.014207124710083, + "rewards/rejected": -2.039699077606201, + "step": 2160 + }, + { + "epoch": 0.7747233131024635, + "grad_norm": 4.827226161956787, + "learning_rate": 1.126383434487683e-05, + "logits/chosen": 19.71803092956543, + "logits/rejected": 19.300630569458008, + "logps/chosen": -55.87500762939453, + "logps/rejected": -42.52462387084961, + "loss": 0.4894, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.7345470190048218, + "rewards/margins": 0.6044321656227112, + "rewards/rejected": -1.3389792442321777, + "step": 2170 + }, + { + "epoch": 0.7782934666190646, + "grad_norm": 9.447345733642578, + "learning_rate": 1.108532666904677e-05, + "logits/chosen": 19.782794952392578, + "logits/rejected": 19.624738693237305, + "logps/chosen": -45.2284049987793, + "logps/rejected": -68.25267791748047, + "loss": 0.443, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.21583375334739685, + "rewards/margins": 0.9129629135131836, + "rewards/rejected": -1.1287966966629028, + "step": 2180 + }, + { + "epoch": 0.7818636201356658, + "grad_norm": 6.821338176727295, + "learning_rate": 1.090681899321671e-05, + "logits/chosen": 22.291385650634766, + "logits/rejected": 22.491024017333984, + "logps/chosen": -35.10066223144531, + "logps/rejected": -53.37052536010742, + "loss": 0.4743, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.867012619972229, + "rewards/margins": 0.7674096822738647, + "rewards/rejected": -1.6344223022460938, + "step": 2190 + }, + { + "epoch": 0.7854337736522671, + "grad_norm": 13.862120628356934, + "learning_rate": 1.0728311317386647e-05, + "logits/chosen": 24.0218448638916, + "logits/rejected": 23.70220947265625, + "logps/chosen": -70.60053253173828, + "logps/rejected": -49.052589416503906, + "loss": 0.5865, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9927393794059753, + "rewards/margins": 0.9688960313796997, + "rewards/rejected": -1.9616353511810303, + "step": 2200 + }, + { + "epoch": 0.7854337736522671, + "eval_logits/chosen": 20.780187606811523, + "eval_logits/rejected": 20.791622161865234, + "eval_logps/chosen": -47.48575210571289, + "eval_logps/rejected": -52.499168395996094, + "eval_loss": 0.43738028407096863, + "eval_rewards/accuracies": 0.8166666626930237, + "eval_rewards/chosen": -0.5845859050750732, + "eval_rewards/margins": 1.4616053104400635, + "eval_rewards/rejected": -2.0461909770965576, + "eval_runtime": 97.9018, + "eval_samples_per_second": 14.627, + "eval_steps_per_second": 0.919, + "step": 2200 + }, + { + "epoch": 0.7890039271688682, + "grad_norm": 9.391676902770996, + "learning_rate": 1.0549803641556586e-05, + "logits/chosen": 19.061254501342773, + "logits/rejected": 18.95170021057129, + "logps/chosen": -24.127471923828125, + "logps/rejected": -29.48319435119629, + "loss": 0.3811, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.1544402837753296, + "rewards/margins": 0.755560040473938, + "rewards/rejected": -1.9100004434585571, + "step": 2210 + }, + { + "epoch": 0.7925740806854695, + "grad_norm": 15.025814056396484, + "learning_rate": 1.0371295965726527e-05, + "logits/chosen": 22.466449737548828, + "logits/rejected": 22.482467651367188, + "logps/chosen": -66.04009246826172, + "logps/rejected": -77.5560073852539, + "loss": 0.5311, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.26618120074272156, + "rewards/margins": 1.5414941310882568, + "rewards/rejected": -1.8076753616333008, + "step": 2220 + }, + { + "epoch": 0.7961442342020707, + "grad_norm": 4.339944839477539, + "learning_rate": 1.0192788289896466e-05, + "logits/chosen": 22.064773559570312, + "logits/rejected": 22.116168975830078, + "logps/chosen": -46.800048828125, + "logps/rejected": -47.54998016357422, + "loss": 0.4871, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.3751857280731201, + "rewards/margins": 0.8837114572525024, + "rewards/rejected": -2.258897304534912, + "step": 2230 + }, + { + "epoch": 0.7997143877186719, + "grad_norm": 6.989345550537109, + "learning_rate": 1.0014280614066405e-05, + "logits/chosen": 21.530237197875977, + "logits/rejected": 21.780250549316406, + "logps/chosen": -20.928579330444336, + "logps/rejected": -34.38627243041992, + "loss": 0.3897, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6259101629257202, + "rewards/margins": 1.1122276782989502, + "rewards/rejected": -1.7381378412246704, + "step": 2240 + }, + { + "epoch": 0.8032845412352732, + "grad_norm": 10.125934600830078, + "learning_rate": 9.835772938236344e-06, + "logits/chosen": 19.070201873779297, + "logits/rejected": 19.056577682495117, + "logps/chosen": -57.353233337402344, + "logps/rejected": -45.93163299560547, + "loss": 0.6086, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.8172680735588074, + "rewards/margins": 1.0970861911773682, + "rewards/rejected": -1.9143543243408203, + "step": 2250 + }, + { + "epoch": 0.8068546947518743, + "grad_norm": 3.600658655166626, + "learning_rate": 9.657265262406285e-06, + "logits/chosen": 25.529754638671875, + "logits/rejected": 25.521421432495117, + "logps/chosen": -34.79078674316406, + "logps/rejected": -43.617095947265625, + "loss": 0.513, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.6766828894615173, + "rewards/margins": -0.25649863481521606, + "rewards/rejected": -0.42018431425094604, + "step": 2260 + }, + { + "epoch": 0.8104248482684755, + "grad_norm": 9.139204025268555, + "learning_rate": 9.478757586576224e-06, + "logits/chosen": 19.52242660522461, + "logits/rejected": 19.63443946838379, + "logps/chosen": -33.019832611083984, + "logps/rejected": -49.0518684387207, + "loss": 0.4795, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.5696147084236145, + "rewards/margins": 1.7224088907241821, + "rewards/rejected": -2.2920238971710205, + "step": 2270 + }, + { + "epoch": 0.8139950017850768, + "grad_norm": 8.38289737701416, + "learning_rate": 9.300249910746163e-06, + "logits/chosen": 21.76547622680664, + "logits/rejected": 22.202754974365234, + "logps/chosen": -57.740699768066406, + "logps/rejected": -56.75148391723633, + "loss": 0.3864, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.994134247303009, + "rewards/margins": 1.7650930881500244, + "rewards/rejected": -2.7592272758483887, + "step": 2280 + }, + { + "epoch": 0.817565155301678, + "grad_norm": 7.5334272384643555, + "learning_rate": 9.121742234916102e-06, + "logits/chosen": 24.34808349609375, + "logits/rejected": 24.858325958251953, + "logps/chosen": -25.712127685546875, + "logps/rejected": -42.50498962402344, + "loss": 0.6181, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.5895062685012817, + "rewards/margins": 1.4605395793914795, + "rewards/rejected": -2.0500457286834717, + "step": 2290 + }, + { + "epoch": 0.8211353088182792, + "grad_norm": 6.730588912963867, + "learning_rate": 8.943234559086042e-06, + "logits/chosen": 23.88949203491211, + "logits/rejected": 23.867450714111328, + "logps/chosen": -29.585073471069336, + "logps/rejected": -44.84501647949219, + "loss": 0.4544, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.01124107837677, + "rewards/margins": 1.032428503036499, + "rewards/rejected": -2.0436697006225586, + "step": 2300 + }, + { + "epoch": 0.8211353088182792, + "eval_logits/chosen": 20.524213790893555, + "eval_logits/rejected": 20.531003952026367, + "eval_logps/chosen": -47.56471252441406, + "eval_logps/rejected": -52.853515625, + "eval_loss": 0.4356454610824585, + "eval_rewards/accuracies": 0.8333333134651184, + "eval_rewards/chosen": -0.5924822688102722, + "eval_rewards/margins": 1.4891438484191895, + "eval_rewards/rejected": -2.0816261768341064, + "eval_runtime": 97.6895, + "eval_samples_per_second": 14.659, + "eval_steps_per_second": 0.921, + "step": 2300 + }, + { + "epoch": 0.8247054623348804, + "grad_norm": 7.662333011627197, + "learning_rate": 8.76472688325598e-06, + "logits/chosen": 19.167724609375, + "logits/rejected": 19.311237335205078, + "logps/chosen": -60.642913818359375, + "logps/rejected": -61.336265563964844, + "loss": 0.4462, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.7695547342300415, + "rewards/margins": 1.0697910785675049, + "rewards/rejected": -1.839345932006836, + "step": 2310 + }, + { + "epoch": 0.8282756158514816, + "grad_norm": 13.9297513961792, + "learning_rate": 8.58621920742592e-06, + "logits/chosen": 24.825382232666016, + "logits/rejected": 24.4403076171875, + "logps/chosen": -50.69221115112305, + "logps/rejected": -43.589820861816406, + "loss": 0.472, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.14803439378738403, + "rewards/margins": 2.013376474380493, + "rewards/rejected": -2.1614108085632324, + "step": 2320 + }, + { + "epoch": 0.8318457693680829, + "grad_norm": 7.068613529205322, + "learning_rate": 8.40771153159586e-06, + "logits/chosen": 20.52377700805664, + "logits/rejected": 20.742507934570312, + "logps/chosen": -48.990299224853516, + "logps/rejected": -56.431175231933594, + "loss": 0.5638, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6809730529785156, + "rewards/margins": 1.5615565776824951, + "rewards/rejected": -2.2425293922424316, + "step": 2330 + }, + { + "epoch": 0.835415922884684, + "grad_norm": 6.40026330947876, + "learning_rate": 8.229203855765798e-06, + "logits/chosen": 20.41210174560547, + "logits/rejected": 20.379867553710938, + "logps/chosen": -59.4276123046875, + "logps/rejected": -71.07743072509766, + "loss": 0.5384, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.5872411727905273, + "rewards/margins": 0.9024609327316284, + "rewards/rejected": -2.4897022247314453, + "step": 2340 + }, + { + "epoch": 0.8389860764012853, + "grad_norm": 9.892749786376953, + "learning_rate": 8.050696179935737e-06, + "logits/chosen": 22.273807525634766, + "logits/rejected": 22.05521011352539, + "logps/chosen": -37.72688293457031, + "logps/rejected": -37.593387603759766, + "loss": 0.4539, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.0601468086242676, + "rewards/margins": 0.7513321042060852, + "rewards/rejected": -1.811478853225708, + "step": 2350 + }, + { + "epoch": 0.8425562299178865, + "grad_norm": 19.193857192993164, + "learning_rate": 7.872188504105676e-06, + "logits/chosen": 21.529739379882812, + "logits/rejected": 21.197284698486328, + "logps/chosen": -25.73797607421875, + "logps/rejected": -45.44214630126953, + "loss": 0.5542, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.01898249424993992, + "rewards/margins": 1.9577863216400146, + "rewards/rejected": -1.9388039112091064, + "step": 2360 + }, + { + "epoch": 0.8461263834344876, + "grad_norm": 9.629522323608398, + "learning_rate": 7.693680828275617e-06, + "logits/chosen": 13.054910659790039, + "logits/rejected": 12.92231559753418, + "logps/chosen": -28.6776180267334, + "logps/rejected": -39.813411712646484, + "loss": 0.5216, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.1042935848236084, + "rewards/margins": 1.2019240856170654, + "rewards/rejected": -2.306217670440674, + "step": 2370 + }, + { + "epoch": 0.8496965369510889, + "grad_norm": 8.702773094177246, + "learning_rate": 7.515173152445555e-06, + "logits/chosen": 24.35626792907715, + "logits/rejected": 24.593463897705078, + "logps/chosen": -44.167537689208984, + "logps/rejected": -70.86328125, + "loss": 0.4955, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.1845743656158447, + "rewards/margins": 1.8204050064086914, + "rewards/rejected": -3.004979372024536, + "step": 2380 + }, + { + "epoch": 0.8532666904676901, + "grad_norm": 11.280304908752441, + "learning_rate": 7.336665476615495e-06, + "logits/chosen": 21.997047424316406, + "logits/rejected": 22.113534927368164, + "logps/chosen": -57.37868118286133, + "logps/rejected": -64.85008239746094, + "loss": 0.5263, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.2687997817993164, + "rewards/margins": 0.5181077718734741, + "rewards/rejected": -1.7869075536727905, + "step": 2390 + }, + { + "epoch": 0.8568368439842913, + "grad_norm": 7.5682854652404785, + "learning_rate": 7.158157800785434e-06, + "logits/chosen": 23.916316986083984, + "logits/rejected": 23.814477920532227, + "logps/chosen": -60.103416442871094, + "logps/rejected": -71.51342010498047, + "loss": 0.4501, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.49635568261146545, + "rewards/margins": 1.9991073608398438, + "rewards/rejected": -2.4954631328582764, + "step": 2400 + }, + { + "epoch": 0.8568368439842913, + "eval_logits/chosen": 20.50776481628418, + "eval_logits/rejected": 20.51170539855957, + "eval_logps/chosen": -47.33561706542969, + "eval_logps/rejected": -52.59519958496094, + "eval_loss": 0.43392252922058105, + "eval_rewards/accuracies": 0.8444444537162781, + "eval_rewards/chosen": -0.5695720911026001, + "eval_rewards/margins": 1.4862221479415894, + "eval_rewards/rejected": -2.0557942390441895, + "eval_runtime": 97.6403, + "eval_samples_per_second": 14.666, + "eval_steps_per_second": 0.922, + "step": 2400 + }, + { + "epoch": 0.8604069975008926, + "grad_norm": 5.012470722198486, + "learning_rate": 6.979650124955374e-06, + "logits/chosen": 21.85761260986328, + "logits/rejected": 21.804880142211914, + "logps/chosen": -31.270885467529297, + "logps/rejected": -52.126930236816406, + "loss": 0.478, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.35933440923690796, + "rewards/margins": 1.06755530834198, + "rewards/rejected": -1.4268897771835327, + "step": 2410 + }, + { + "epoch": 0.8639771510174937, + "grad_norm": 5.9783854484558105, + "learning_rate": 6.801142449125313e-06, + "logits/chosen": 19.979433059692383, + "logits/rejected": 20.035179138183594, + "logps/chosen": -51.7042121887207, + "logps/rejected": -57.767913818359375, + "loss": 0.4853, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.6708565354347229, + "rewards/margins": 1.6906753778457642, + "rewards/rejected": -2.3615317344665527, + "step": 2420 + }, + { + "epoch": 0.867547304534095, + "grad_norm": 7.05979061126709, + "learning_rate": 6.6226347732952526e-06, + "logits/chosen": 15.46339225769043, + "logits/rejected": 15.658937454223633, + "logps/chosen": -40.01195526123047, + "logps/rejected": -46.074501037597656, + "loss": 0.5628, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.18836592137813568, + "rewards/margins": 1.5064311027526855, + "rewards/rejected": -1.6947968006134033, + "step": 2430 + }, + { + "epoch": 0.8711174580506962, + "grad_norm": 12.882414817810059, + "learning_rate": 6.4441270974651915e-06, + "logits/chosen": 21.048954010009766, + "logits/rejected": 21.059772491455078, + "logps/chosen": -53.65748977661133, + "logps/rejected": -48.547874450683594, + "loss": 0.4885, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -1.3018702268600464, + "rewards/margins": 1.0138559341430664, + "rewards/rejected": -2.3157262802124023, + "step": 2440 + }, + { + "epoch": 0.8746876115672974, + "grad_norm": 11.930545806884766, + "learning_rate": 6.26561942163513e-06, + "logits/chosen": 20.654542922973633, + "logits/rejected": 20.686420440673828, + "logps/chosen": -39.971920013427734, + "logps/rejected": -35.28145217895508, + "loss": 0.5033, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.6068005561828613, + "rewards/margins": 1.3182332515716553, + "rewards/rejected": -1.9250338077545166, + "step": 2450 + }, + { + "epoch": 0.8782577650838986, + "grad_norm": 9.403008460998535, + "learning_rate": 6.08711174580507e-06, + "logits/chosen": 22.882566452026367, + "logits/rejected": 22.231182098388672, + "logps/chosen": -52.44697952270508, + "logps/rejected": -47.005287170410156, + "loss": 0.5709, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.8271516561508179, + "rewards/margins": 0.7867218852043152, + "rewards/rejected": -1.6138734817504883, + "step": 2460 + }, + { + "epoch": 0.8818279186004998, + "grad_norm": 23.268875122070312, + "learning_rate": 5.908604069975009e-06, + "logits/chosen": 21.904462814331055, + "logits/rejected": 22.555965423583984, + "logps/chosen": -63.27705001831055, + "logps/rejected": -87.00935363769531, + "loss": 0.383, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.2159323692321777, + "rewards/margins": 1.8115613460540771, + "rewards/rejected": -3.027493715286255, + "step": 2470 + }, + { + "epoch": 0.8853980721171011, + "grad_norm": 16.253660202026367, + "learning_rate": 5.730096394144948e-06, + "logits/chosen": 22.083019256591797, + "logits/rejected": 22.175024032592773, + "logps/chosen": -43.36615753173828, + "logps/rejected": -42.4653205871582, + "loss": 0.531, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.2614014148712158, + "rewards/margins": 0.07179747521877289, + "rewards/rejected": -1.33319890499115, + "step": 2480 + }, + { + "epoch": 0.8889682256337023, + "grad_norm": 8.766143798828125, + "learning_rate": 5.551588718314888e-06, + "logits/chosen": 22.502155303955078, + "logits/rejected": 22.834453582763672, + "logps/chosen": -57.746620178222656, + "logps/rejected": -73.5748519897461, + "loss": 0.386, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6636447906494141, + "rewards/margins": 1.0091074705123901, + "rewards/rejected": -1.6727523803710938, + "step": 2490 + }, + { + "epoch": 0.8925383791503034, + "grad_norm": 9.421289443969727, + "learning_rate": 5.373081042484827e-06, + "logits/chosen": 22.923538208007812, + "logits/rejected": 23.32149887084961, + "logps/chosen": -63.822181701660156, + "logps/rejected": -57.5202522277832, + "loss": 0.5171, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.152776837348938, + "rewards/margins": 0.4619804322719574, + "rewards/rejected": -1.6147572994232178, + "step": 2500 + }, + { + "epoch": 0.8925383791503034, + "eval_logits/chosen": 20.416898727416992, + "eval_logits/rejected": 20.427398681640625, + "eval_logps/chosen": -47.32997131347656, + "eval_logps/rejected": -52.617210388183594, + "eval_loss": 0.4359460473060608, + "eval_rewards/accuracies": 0.8333333134651184, + "eval_rewards/chosen": -0.5690072774887085, + "eval_rewards/margins": 1.4889880418777466, + "eval_rewards/rejected": -2.057995319366455, + "eval_runtime": 98.594, + "eval_samples_per_second": 14.524, + "eval_steps_per_second": 0.913, + "step": 2500 + }, + { + "epoch": 0.8961085326669047, + "grad_norm": 10.914727210998535, + "learning_rate": 5.194573366654766e-06, + "logits/chosen": 27.528060913085938, + "logits/rejected": 27.470294952392578, + "logps/chosen": -26.07723045349121, + "logps/rejected": -25.864681243896484, + "loss": 0.4914, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3140444755554199, + "rewards/margins": 0.8342065811157227, + "rewards/rejected": -1.1482508182525635, + "step": 2510 + }, + { + "epoch": 0.8996786861835059, + "grad_norm": 4.41356897354126, + "learning_rate": 5.016065690824705e-06, + "logits/chosen": 21.382648468017578, + "logits/rejected": 21.26772689819336, + "logps/chosen": -65.88087463378906, + "logps/rejected": -38.547142028808594, + "loss": 0.5372, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.7923516631126404, + "rewards/margins": 0.6893059015274048, + "rewards/rejected": -1.4816573858261108, + "step": 2520 + }, + { + "epoch": 0.903248839700107, + "grad_norm": 14.50999641418457, + "learning_rate": 4.837558014994645e-06, + "logits/chosen": 24.574230194091797, + "logits/rejected": 24.432174682617188, + "logps/chosen": -40.37361145019531, + "logps/rejected": -61.320648193359375, + "loss": 0.59, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.6994588971138, + "rewards/margins": 1.3289554119110107, + "rewards/rejected": -2.028414487838745, + "step": 2530 + }, + { + "epoch": 0.9068189932167083, + "grad_norm": 3.6535849571228027, + "learning_rate": 4.659050339164584e-06, + "logits/chosen": 22.09380340576172, + "logits/rejected": 22.342832565307617, + "logps/chosen": -43.704811096191406, + "logps/rejected": -57.10416793823242, + "loss": 0.3651, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.5590577721595764, + "rewards/margins": 2.217897891998291, + "rewards/rejected": -2.7769553661346436, + "step": 2540 + }, + { + "epoch": 0.9103891467333095, + "grad_norm": 12.662737846374512, + "learning_rate": 4.480542663334524e-06, + "logits/chosen": 26.030466079711914, + "logits/rejected": 26.27060317993164, + "logps/chosen": -45.546714782714844, + "logps/rejected": -56.18561935424805, + "loss": 0.631, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6547432541847229, + "rewards/margins": 1.1012141704559326, + "rewards/rejected": -1.7559573650360107, + "step": 2550 + }, + { + "epoch": 0.9139593002499108, + "grad_norm": 12.380247116088867, + "learning_rate": 4.302034987504463e-06, + "logits/chosen": 26.792774200439453, + "logits/rejected": 26.5482120513916, + "logps/chosen": -56.17375564575195, + "logps/rejected": -41.74138641357422, + "loss": 0.5118, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.4354368448257446, + "rewards/margins": 0.7226849794387817, + "rewards/rejected": -2.1581218242645264, + "step": 2560 + }, + { + "epoch": 0.917529453766512, + "grad_norm": 14.337486267089844, + "learning_rate": 4.123527311674403e-06, + "logits/chosen": 20.274442672729492, + "logits/rejected": 20.080303192138672, + "logps/chosen": -64.95756530761719, + "logps/rejected": -54.136146545410156, + "loss": 0.4504, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9958245158195496, + "rewards/margins": 1.3853988647460938, + "rewards/rejected": -2.381223440170288, + "step": 2570 + }, + { + "epoch": 0.9210996072831131, + "grad_norm": 5.5064616203308105, + "learning_rate": 3.945019635844342e-06, + "logits/chosen": 23.955215454101562, + "logits/rejected": 23.814537048339844, + "logps/chosen": -47.43402862548828, + "logps/rejected": -75.19926452636719, + "loss": 0.3622, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.0230860710144043, + "rewards/margins": 1.3491160869598389, + "rewards/rejected": -2.372201919555664, + "step": 2580 + }, + { + "epoch": 0.9246697607997144, + "grad_norm": 8.065810203552246, + "learning_rate": 3.766511960014281e-06, + "logits/chosen": 24.445186614990234, + "logits/rejected": 24.539066314697266, + "logps/chosen": -54.058349609375, + "logps/rejected": -57.293846130371094, + "loss": 0.5582, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.8545524477958679, + "rewards/margins": 0.8429762721061707, + "rewards/rejected": -1.697528600692749, + "step": 2590 + }, + { + "epoch": 0.9282399143163156, + "grad_norm": 8.037162780761719, + "learning_rate": 3.5880042841842204e-06, + "logits/chosen": 22.97157859802246, + "logits/rejected": 22.82067108154297, + "logps/chosen": -36.165348052978516, + "logps/rejected": -39.38242721557617, + "loss": 0.4775, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -1.074142336845398, + "rewards/margins": 0.5587809681892395, + "rewards/rejected": -1.6329231262207031, + "step": 2600 + }, + { + "epoch": 0.9282399143163156, + "eval_logits/chosen": 20.513593673706055, + "eval_logits/rejected": 20.519195556640625, + "eval_logps/chosen": -47.434913635253906, + "eval_logps/rejected": -52.493404388427734, + "eval_loss": 0.43444961309432983, + "eval_rewards/accuracies": 0.8333333134651184, + "eval_rewards/chosen": -0.5795022249221802, + "eval_rewards/margins": 1.4661126136779785, + "eval_rewards/rejected": -2.045614719390869, + "eval_runtime": 97.6912, + "eval_samples_per_second": 14.658, + "eval_steps_per_second": 0.921, + "step": 2600 + } + ], + "logging_steps": 10, + "max_steps": 2801, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 100, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}